def test_roundtrip():
point = (12, 5)
trans = Affine.translation(3, 4)
rot37 = Affine.rotation(37.)
point_prime = (trans * rot37) * point
roundtrip_point = ~(trans * rot37) * point_prime
seq_almost_equal(point, roundtrip_point)
def from_geopolygon(cls, geopolygon, resolution, crs=None, align=None):
"""
:type geopolygon: GeoPolygon
:param resolution: (x_resolution, y_resolution)
:param CRS crs: CRS to use, if different from the geopolygon
:param (float,float) align: Alight geobox such that point 'align' lies on the pixel boundary.
:rtype: GeoBox
"""
# TODO: currently only flipped Y-axis data is supported
assert resolution[1] > 0
assert resolution[0] < 0
align = align or (0.0, 0.0)
assert 0.0 <= align[1] <= abs(resolution[1])
assert 0.0 <= align[0] <= abs(resolution[0])
if crs is None:
crs = geopolygon.crs
else:
geopolygon = geopolygon.to_crs(crs)
def align_pix(val, res, off):
return math.floor((val-off)/res) * res + off
bounding_box = geopolygon.boundingbox
left = align_pix(bounding_box.left, resolution[1], align[1])
top = align_pix(bounding_box.top, resolution[0], align[0])
affine = (Affine.translation(left, top) * Affine.scale(resolution[1], resolution[0]))
return GeoBox(crs=crs,
affine=affine,
width=int(math.ceil((bounding_box.right-left)/resolution[1])),
height=int(math.ceil((bounding_box.bottom-top)/resolution[0])))
def explore(self, T0, *args):
"""
Evaluate the similarity at the transformations specified by
sequences of parameter values.
For instance:
explore(T0, (0, [-1,0,1]), (4, [-2.,2]))
"""
nparams = T0.param.size
sizes = np.ones(nparams)
deltas = [[0] for i in range(nparams)]
for a in args:
deltas[a[0]] = a[1]
grids = np.mgrid[[slice(0, len(d)) for d in deltas]]
ntrials = np.prod(grids.shape[1:])
Deltas = [np.asarray(deltas[i])[grids[i,:]].ravel() for i in range(nparams)]
simis = np.zeros(ntrials)
params = np.zeros([nparams, ntrials])
T = Affine()
for i in range(ntrials):
t = T0.param + np.array([D[i] for D in Deltas])
T.param = t
simis[i] = self.eval(T)
params[:, i] = t
return simis, params
def transform_from_latlon( lat, lon ): ''' simple way to make an affine transform from lats and lons coords ''' from affine import Affine lat = np.asarray( lat ) lon = np.asarray(lon) trans = Affine.translation(lon[0], lat[0]) scale = Affine.scale(lon[1] - lon[0], lat[1] - lat[0]) return trans * scale
def test_scale_constructor(self):
scale = Affine.scale(5)
assert isinstance(scale, Affine)
assert_equal(tuple(scale), (5,0,0, 0,5,0, 0,0,1))
scale = Affine.scale(-1, 2)
assert_equal(tuple(scale), (-1,0,0, 0,2,0, 0,0,1))
assert_equal(tuple(Affine.scale(1)),
tuple(Affine.identity()))
def from_origin(west, north, xsize, ysize):
"""Return an Affine transformation given upper left and pixel sizes.
Return an Affine transformation for a georeferenced raster given
the coordinates of its upper left corner `west`, `north` and pixel
sizes `xsize`, `ysize`.
"""
return Affine.translation(west, north) * Affine.scale(xsize, -ysize)
def test_rotation_matrix_pivot():
"""A rotation matrix with pivot has expected elements"""
rot = Affine.rotation(90.0, pivot=(1.0, 1.0))
exp = (Affine.translation(1.0, 1.0)
* Affine.rotation(90.0)
* Affine.translation(-1.0, -1.0))
for r, e in zip(rot, exp):
assert round(r, 15) == round(e, 15)
def test_associative():
point = (12, 5)
trans = Affine.translation(-10., -5.)
rot90 = Affine.rotation(90.)
result1 = rot90 * (trans * point)
result2 = (rot90 * trans) * point
seq_almost_equal(result1, (0., 2.))
seq_almost_equal(result1, result2)
def test_reproject_dst_crs_none():
with pytest.raises(CRSError):
reproject(
np.ones((2, 2)),
np.zeros((2, 2)),
src_transform=Affine.identity(),
dst_transform=Affine.identity(),
src_crs=WGS84_crs,
)
def from_bounds(west, south, east, north, width, height):
"""Return an Affine transformation given bounds, width and height.
Return an Affine transformation for a georeferenced raster given
its bounds `west`, `south`, `east`, `north` and its `width` and
`height` in number of pixels.
"""
return Affine.translation(west, north) * Affine.scale(
(east - west) / width, (south - north) / height)
def affine(self, pixelbuffer=0):
"""
Return an Affine object of tile.
- pixelbuffer: tile buffer in pixels
"""
left = self.bounds(pixelbuffer=pixelbuffer)[0]
top = self.bounds(pixelbuffer=pixelbuffer)[3]
return Affine.translation(left, top) * Affine.scale(
self.pixel_x_size, -self.pixel_y_size)
def test_rotation_constructor_with_pivot(self):
assert_equal(tuple(Affine.rotation(60)),
tuple(Affine.rotation(60, pivot=(0,0))))
rot = Affine.rotation(27, pivot=(2,-4))
r = math.radians(27)
s, c = math.sin(r), math.cos(r)
assert_equal(tuple(rot),
(c,s,2 - 2*c - 4*s, -s,c,-4 - 2*s + 4*c, 0,0,1))
assert_equal(tuple(Affine.rotation(0, (-3, 2))),
tuple(Affine.identity()))
def test_is_degenerate(self):
assert not Affine.identity().is_degenerate
assert not Affine.translation(2, -1).is_degenerate
assert not Affine.shear(0, -22.5).is_degenerate
assert not Affine.rotation(88.7).is_degenerate
assert not Affine.scale(0.5).is_degenerate
assert Affine.scale(0).is_degenerate
assert Affine.scale(-10, 0).is_degenerate
assert Affine.scale(0, 300).is_degenerate
assert Affine.scale(0).is_degenerate
assert Affine.scale(0).is_degenerate
def test_almost_equals(self):
EPSILON = 1e-5
E = EPSILON * 0.5
t = Affine(1.0, E, 0, -E, 1.0 + E, E)
assert t.almost_equals(Affine.identity())
assert Affine.identity().almost_equals(t)
assert t.almost_equals(t)
t = Affine(1.0, 0, 0, -EPSILON, 1.0, 0)
assert not t.almost_equals(Affine.identity())
assert not Affine.identity().almost_equals(t)
assert t.almost_equals(t)
def test_almost_equals_2(self):
EPSILON = 1e-10
E = EPSILON * 0.5
t = Affine(1.0, E, 0, -E, 1.0 + E, E)
assert t.almost_equals(Affine.identity(), precision=EPSILON)
assert Affine.identity().almost_equals(t, precision=EPSILON)
assert t.almost_equals(t, precision=EPSILON)
t = Affine(1.0, 0, 0, -EPSILON, 1.0, 0)
assert not t.almost_equals(Affine.identity(), precision=EPSILON)
assert not Affine.identity().almost_equals(t, precision=EPSILON)
assert t.almost_equals(t, precision=EPSILON)
def open(path, mode='r', width=None, height=None, count=None, transform=None,
crs=None, no_data=None, dtype=None, chunks=(256, 256),
blocksize=256, compression=1, band_names=None):
# should we pass to different read write classes based on the mode?
if mode == 'r':
fid = h5py.File(path, mode)
ds = KeaH5RDOnly(fid)
elif mode == 'r+':
fid = h5py.File(path, mode)
ds = KeaH5RW(fid)
elif mode =='w':
# Check we have all the necessary creation options
if (width is None) or (height is None):
msg = "Error. Both width and height must be specified."
raise ValueError(msg)
if dtype is None:
msg = "Error. The dtype must be specifified."
raise ValueError(msg)
if count is None:
msg = "Error. The count must be specified."
raise ValueError(msg)
# If we have no transform, default to image co-ordinates
if (transform is None) or (crs is None):
ul = (0, 0)
rot = (0, 0)
res = (1, -1)
transform = Affine.from_gdal(*[0.0, 1.0, 0.0, 0.0, 0.0, -1.0])
crs = ""
if (chunks[0] > height) or (chunks[1] > width):
msg = "The chunks must not exceed the width or height."
raise ValueError(msg)
# we'll use rasterio's proj4 dict mapping
if not isinstance(crs, dict):
msg = "Error. The crs is not a valid proj4 dict style mapping."
raise ValueError(msg)
# we'll follow rasterio in using an affine
if not isinstance(transform, Affine):
msg = "Error. The transform is not an Affine instance."
transform = Affine.from_gdal(*transform)
fid = h5py.File(path, mode)
create_kea_image(fid, width, height, count, transform, crs, no_data,
dtype, chunks, blocksize, compression, band_names)
ds = KeaH5RW(fid)
return ds
def test_shear_constructor(self):
shear = Affine.shear(30)
assert isinstance(shear, Affine)
sx = math.tan(math.radians(30))
seq_almost_equal(tuple(shear), (1,0,0, sx,1,0, 0,0,1))
shear = Affine.shear(-15, 60)
sx = math.tan(math.radians(-15))
sy = math.tan(math.radians(60))
seq_almost_equal(tuple(shear), (1,sy,0, sx,1,0, 0,0,1))
shear = Affine.shear(y_angle=45)
seq_almost_equal(tuple(shear), (1,1,0, 0,1,0, 0,0,1))
def test_itransform(self):
pts = [(4, 1), (-1, 0), (3, 2)]
r = Affine.scale(-2).itransform(pts)
assert r is None, r
assert pts == [(-8, -2), (2, 0), (-6, -4)]
A = Affine.rotation(33)
pts = [(4, 1), (-1, 0), (3, 2)]
pts_expect = [A*pt for pt in pts]
r = A.itransform(pts)
assert r is None
assert pts == pts_expect
def test_rotation_constructor(self):
rot = Affine.rotation(60)
assert isinstance(rot, Affine)
r = math.radians(60)
s, c = math.sin(r), math.cos(r)
assert_equal(tuple(rot), (c,s,0, -s,c,0, 0,0,1))
rot = Affine.rotation(337)
r = math.radians(337)
s, c = math.sin(r), math.cos(r)
seq_almost_equal(tuple(rot), (c,s,0, -s,c,0, 0,0,1))
assert_equal(tuple(Affine.rotation(0)),
tuple(Affine.identity()))
def test_almost_equals(self):
from affine import EPSILON
assert EPSILON != 0, EPSILON
E = EPSILON * 0.5
t = Affine(1.0, E, 0, -E, 1.0 + E, E)
assert t.almost_equals(Affine.identity())
assert Affine.identity().almost_equals(t)
assert t.almost_equals(t)
t = Affine(1.0, 0, 0, -EPSILON, 1.0, 0)
assert not t.almost_equals(Affine.identity())
assert not Affine.identity().almost_equals(t)
assert t.almost_equals(t)
def test_rotation_constructor_quadrants(self):
assert_equal(tuple(Affine.rotation(0)), (1,0,0, 0,1,0, 0,0,1))
assert_equal(tuple(Affine.rotation(90)), (0,1,0, -1,0,0, 0,0,1))
assert_equal(tuple(Affine.rotation(180)), (-1,0,0, 0,-1,0, 0,0,1))
assert_equal(tuple(Affine.rotation(-180)), (-1,0,0, 0,-1,0, 0,0,1))
assert_equal(tuple(Affine.rotation(270)), (0,-1,0, 1,0,0, 0,0,1))
assert_equal(tuple(Affine.rotation(-90)), (0,-1,0, 1,0,0, 0,0,1))
assert_equal(tuple(Affine.rotation(360)), (1,0,0, 0,1,0, 0,0,1))
assert_equal(tuple(Affine.rotation(450)), (0,1,0, -1,0,0, 0,0,1))
assert_equal(tuple(Affine.rotation(-450)), (0,-1,0, 1,0,0, 0,0,1))
def test_scale_constructor(self):
scale = Affine.scale(5)
assert isinstance(scale, Affine)
assert \
tuple(scale) == \
(5, 0, 0,
0, 5, 0,
0, 0, 1)
scale = Affine.scale(-1, 2)
assert \
tuple(scale) == \
(-1, 0, 0,
0, 2, 0,
0, 0, 1)
assert tuple(Affine.scale(1)) == tuple(Affine.identity())
def from_geopolygon(cls, geopolygon, resolution, crs=None, align=True):
"""
:type geopolygon: datacube.model.GeoPolygon
:param resolution: (x_resolution, y_resolution)
:param crs: CRS to use, if different from the geopolygon
:param align: Should the geobox be aligned to pixels of the given resolution. This assumes an origin of (0,0).
:type align: boolean
:rtype: GeoBox
"""
# TODO: currently only flipped Y-axis data is supported
assert resolution[1] > 0
assert resolution[0] < 0
if crs is None:
crs = geopolygon.crs
else:
geopolygon = geopolygon.to_crs(crs)
bounding_box = geopolygon.boundingbox
# print(bounding_box)
# print("~~~")
left, top = float(bounding_box.left), float(bounding_box.top)
if align:
left = math.floor(left / resolution[1]) * resolution[1]
top = math.floor(top / resolution[0]) * resolution[0]
# print(left)
#print(top)
#print(Affine.translation(left, top))
#print(Affine.scale(resolution[1], resolution[0]))
affine = (Affine.translation(left, top) * Affine.scale(resolution[1], resolution[0]))
# print(affine)
right, bottom = float(bounding_box.right), float(bounding_box.bottom)
width, height = ~affine * (right, bottom)
# print(right)
# print(width)
#width = 198
#height = 288
#print(height)
if align:
width = math.ceil(width)
height = math.ceil(height)
return GeoBox(crs=crs,
affine=affine,
width=int(width),
height=int(height))
def __init__(self, gzx, gzy, yld, ff, wind, wd, shear, tob=0, dunits='km', wunits='km/h', shearunits='m/s-km', yunits='kT'):
self.translation = ~Affine.translation(convert_units(gzx, dunits, 'mi'), convert_units(gzy, dunits, 'mi')) # translate coordinates relative to GZ (st. mi)
self.wd = wd # wind direction in degrees (0=N, 90=E, etc.)
self.yld = convert_units(yld, yunits, 'MT') # yield (MT)
self.ff = ff # fission fraction, 0 < ff <= 1.0
self.wind = convert_units(wind, wunits, 'mph') # wind speed (mi/hr)
self.shear = convert_units(shear, shearunits, 'mph/kilofoot') # wind shear in mi/hr-kilofoot
self.tob = tob # time of burst (hrs)
# FORTRAN is ugly in any language
# Store these values in the WSEG10 object to avoid recalculating them
d = np.log(self.yld) + 2.42 # physically meaningless, but occurs twice
# According to Hanifen, "The cloud is initially formed because the nuclear
# fireball vaporizes both the surface of the earth at ground zero and the
# weapon itself. The activity contained in the cloud is both neutron induced
# and fission. After formation, the fireball rises and begins to cool at its
# outer edges faster than the center thereby creating the typical torroidal
# currents associated with the nuclear cloud. WSEG arbitrarily assumes that
# the cloud will rise to a maximum center height within fifteen minutes and
# then stabilize."
self.H_c = 44 + 6.1 * np.log(yld) - 0.205 * abs(d) * d # cloud center height
lnyield = np.log(self.yld)
self.s_0 = np.exp(0.7 + lnyield / 3 - 3.25 / (4.0 + (lnyield + 5.4)**2)) #sigma_0
self.s_02 = self.s_0**2
self.s_h = 0.18 * self.H_c # sigma_h
self.T_c = 1.0573203 * (12 * (self.H_c / 60) - 2.5 * (self.H_c / 60)**2) * (1 - 0.5 * np.exp(-1 * (self.H_c / 25)**2)) # time constant
self.L_0 = wind * self.T_c # L_0, used by g(x)
self.L_02 = self.L_0**2
self.s_x2 = self.s_02 * (self.L_02 + 8 * self.s_02) / (self.L_02 + 2 * self.s_02)
self.s_x = np.sqrt(self.s_x2) # sigma_x
self.L_2 = self.L_02 + 2 * self.s_x2
self.L = np.sqrt(self.L_2) # L
self.n = (ff * self.L_02 + self.s_x2) / (self.L_02 + 0.5 * self.s_x2) # n
self.a_1 = 1 / (1 + ((0.001 * self.H_c * wind) / self.s_0)) # alpha_1
def coordinates( fn ): ''' take a raster file as input and return the centroid coords for each of the grid cells as a pair of numpy 2d arrays (longitude, latitude) ''' import rasterio import numpy as np from affine import Affine from pyproj import Proj, transform # Read raster with rasterio.open(fn) as r: T0 = r.affine # upper-left pixel corner affine transform p1 = Proj(r.crs) A = r.read_band(1) # pixel values # All rows and columns cols, rows = np.meshgrid(np.arange(A.shape[1]), np.arange(A.shape[0])) # Get affine transform for pixel centres T1 = T0 * Affine.translation(0.5, 0.5) # Function to convert pixel row/column index (from 0) to easting/northing at centre rc2en = lambda r, c: (c, r) * T1 # All eastings and northings (there is probably a faster way to do this) eastings, northings = np.vectorize(rc2en, otypes=[np.float, np.float])(rows, cols) # Project all longitudes, latitudes # longs, lats = transform(p1, p1.to_latlong(), eastings, northings) # return longs, lats return eastings, northings
def test_xy_rowcol_inverse():
# TODO this is an ideal candiate for
# property-based testing with hypothesis
aff = Affine.identity()
rows_cols = ([0, 0, 10, 10],
[0, 10, 0, 10])
assert rows_cols == rowcol(aff, *xy(aff, *rows_cols))
def pixelated_image_file(tmpdir, pixelated_image):
"""
A basic raster file with a 10x10 array for testing sieve functions.
Contains data from pixelated_image.
Returns
-------
string
Filename of test raster file
"""
from affine import Affine
import rasterio
image = pixelated_image
outfilename = str(tmpdir.join('pixelated_image.tif'))
kwargs = {
"crs": CRS({'init': 'epsg:4326'}),
"transform": Affine.identity(),
"count": 1,
"dtype": rasterio.uint8,
"driver": "GTiff",
"width": image.shape[1],
"height": image.shape[0],
"nodata": 255
}
with rasterio.open(outfilename, 'w', **kwargs) as out:
out.write(image, indexes=1)
return outfilename
def clip_land_use_raster(land_use_raster, region_shapefile, output_file):
with rasterio.open(land_use_raster) as r:
with fiona.open(region_shapefile) as clipper:
(w, s, e, n) = clipper.bounds
a = r.affine
#TODO: need to transform the affine for new clipping
(min_col, min_row) = map(int, ~a * (w, n))
(max_col, max_row) = map(int, ~a * (e, s))
w2, n2 = a * (min_col, min_row)
new_affine = Affine.from_gdal(w2, 100, 0.0, n2, 0.0, -100)
(height,width) = r.read(1, window = ((min_row, max_row), (min_col, max_col))).shape
profile = r.profile
profile.update({
'transform': new_affine,
'affine': new_affine,
'height': height,
'width': width
})
with rasterio.open(output_file, 'w', **profile) as out:
for i in r.indexes:
clipped = r.read(i, window = ((min_row, max_row), (min_col, max_col)))
#print clipped.shape
out.write(clipped, indexes = i)
def test_data_dir_1(tmpdir):
kwargs = {
"crs": {'init': 'epsg:4326'},
"transform": Affine.from_gdal(-114, 0.2, 0, 46, 0, -0.2),
"count": 4,
"dtype": rasterio.uint8,
"driver": "GTiff",
"width": 10,
"height": 10
}
with rasterio.Env():
with rasterio.open(str(tmpdir.join('b.tif')), 'w', **kwargs) as dst:
data = numpy.zeros((4, 10, 10), dtype=rasterio.uint8)
data[0:3, 0:6, 0:6] = 255
data[3, 0:6, 0:6] = 255
dst.write(data)
with rasterio.open(str(tmpdir.join('a.tif')), 'w', **kwargs) as dst:
data = numpy.zeros((4, 10, 10), dtype=rasterio.uint8)
data[0:3, 4:8, 4:8] = 254
data[3, 4:8, 4:8] = 255
dst.write(data)
return tmpdir
def chop(bands, dst_transform, size, output_file, photometric):
for y in range(0, bands.shape[1], size):
for x in range(0, bands.shape[2], size):
# crop a grid square ('patch')
patch_affine = Affine(*dst_transform) * Affine.translation(x, y)
patch = numpy.zeros((bands.shape[0], size, size), dtype=rasterio.uint16)
my = min(y + size, bands.shape[1])
mx = min(x + size, bands.shape[2])
for i, band in enumerate(bands):
patch[i, 0:my-y, 0:mx-x] = band[y:my, x:mx]
if(numpy.max(patch) > 0):
out = output_file.replace('{x}', str(x)).replace('{y}', str(y))
with rasterio.open(out,
mode='w', driver='GTiff',
width=patch.shape[2],
height=patch.shape[1],
count=patch.shape[0],
dtype=numpy.uint8,
nodata=0,
photometric=photometric,
transform=patch_affine,
crs=projection) as dst:
for i, band in enumerate(patch):
dst.write_band(i + 1, band.astype(rasterio.uint8))
def make_tile(level, tile):
"""
MISSING
:param level:
:param tile:
:return:
"""
# x,y tile indexes
x = tile[0][0]
y = tile[0][1]
def div_by_16(x):
if divmod(x, 16)[1] == 0:
return x
return div_by_16(x - 1)
# put tile in its respective dir
out_dir = out_folder.joinpath(str(level))
if not out_dir.exists():
out_dir.mkdir(exist_ok=True)
size_x = tile[1].width if tile[1].width > 0 else 1
size_y = tile[1].height if tile[1].height > 0 else 1
# Out file constructor
# how many chars to use for representing the tiles.
name_length = max(len(str(self.tileinfos[level].countTilesX)),
len(str(self.tileinfos[level].countTilesY))) + 1
filename = name_template.format(basename=self.name,
x=str(x).zfill(name_length),
y=str(y).zfill(name_length))
out_filepath = out_dir.joinpath(filename)
## End
profile = default_gtiff_profile
profile.update(
crs='epsg:4326',
driver='GTiff',
transform=tile[2],
compress='lzw',
count=1,
width=size_x,
height=size_y,
blockysize=div_by_16(min(self.blockSize, tile[1].height)),
blockxsize=div_by_16(min(self.blockSize, tile[1].width)),
)
if level > 1:
# except OSError:
# # in this level, the amount of pixels that need to be resampled are too many.
# # I am choosing to use pixel at the central coordinate of the processing tile
# # Sample error:
# # ERROR 1: Integer overflow : nSrcXSize=425985, nSrcYSize=163840
# TODO: don't be lazy, clean write
try:
self.tileinfos[level - 1]
except KeyError:
_meta = self.get_metadata(level - 1)
self.tileinfos[level - 1] = TileInfo(
_meta['width'], _meta['height'], self.TileWidth,
self.TileHeight)
finally:
name_length = max(
len(str(self.tileinfos[level - 1].countTilesX)),
len(str(self.tileinfos[level - 1].countTilesY))) + 1
prev_lvl_tiles = tile_children(zoom=level,
src=out_filepath,
ndigits=name_length)
vrt_handler = buildvrt(prev_lvl_tiles)
with rio.open(vrt_handler) as src:
profile.update(nodata=src.nodata, dtype=src.meta['dtype'])
resolution_factor = pow(2, 1)
lvlx_height = src.height / 2
lvlx_width = src.width / 2
lvlx_tranform = Affine(src.transform.a * resolution_factor,
src.transform.b, src.transform.c,
src.transform.d,
src.transform.e * resolution_factor,
src.transform.f)
vrt = WarpedVRT(src,
transform=lvlx_tranform,
width=lvlx_width,
height=lvlx_height)
data = vrt.read(1)
else:
with self.get_dataset(level) as src:
profile.update(nodata=src.nodata, dtype=src.meta['dtype'])
data = src.read(1, window=tile[1])
try:
with rio.open(out_filepath, 'w', **profile) as dst:
window_out = Window(0, 0, size_x, size_y)
dst.write(data, window=window_out, indexes=1)
except:
print(profile)
raise Exception
def gen_zonal_stats(vectors,
raster,
layer=0,
band=1,
nodata=None,
affine=None,
stats=None,
all_touched=False,
categorical=False,
category_map=None,
add_stats=None,
zone_func=None,
raster_out=False,
prefix=None,
geojson_out=False,
**kwargs):
"""Zonal statistics of raster values aggregated to vector geometries.
Parameters
----------
vectors: path to an vector source or geo-like python objects
raster: ndarray or path to a GDAL raster source
If ndarray is passed, the ``affine`` kwarg is required.
layer: int or string, optional
If `vectors` is a path to an fiona source,
specify the vector layer to use either by name or number.
defaults to 0
band: int, optional
If `raster` is a GDAL source, the band number to use (counting from 1).
defaults to 1.
nodata: float, optional
If `raster` is a GDAL source, this value overrides any NODATA value
specified in the file's metadata.
If `None`, the file's metadata's NODATA value (if any) will be used.
defaults to `None`.
affine: Affine instance
required only for ndarrays, otherwise it is read from src
stats: list of str, or space-delimited str, optional
Which statistics to calculate for each zone.
All possible choices are listed in ``utils.VALID_STATS``.
defaults to ``DEFAULT_STATS``, a subset of these.
all_touched: bool, optional
Whether to include every raster cell touched by a geometry, or only
those having a center point within the polygon.
defaults to `False`
categorical: bool, optional
category_map: dict
A dictionary mapping raster values to human-readable categorical names.
Only applies when categorical is True
add_stats: dict
with names and functions of additional stats to compute, optional
zone_func: callable
function to apply to zone ndarray prior to computing stats
raster_out: boolean
Include the masked numpy array for each feature?, optional
Each feature dictionary will have the following additional keys:
mini_raster_array: The clipped and masked numpy array
mini_raster_affine: Affine transformation
mini_raster_nodata: NoData Value
prefix: string
add a prefix to the keys (default: None)
geojson_out: boolean
Return list of GeoJSON-like features (default: False)
Original feature geometry and properties will be retained
with zonal stats appended as additional properties.
Use with `prefix` to ensure unique and meaningful property names.
Returns
-------
generator of dicts (if geojson_out is False)
Each item corresponds to a single vector feature and
contains keys for each of the specified stats.
generator of geojson features (if geojson_out is True)
GeoJSON-like Feature as python dict
"""
stats, run_count = check_stats(stats, categorical)
# Handle 1.0 deprecations
transform = kwargs.get('transform')
if transform:
warnings.warn(
"GDAL-style transforms will disappear in 1.0. "
"Use affine=Affine.from_gdal(*transform) instead",
DeprecationWarning)
if not affine:
affine = Affine.from_gdal(*transform)
cp = kwargs.get('copy_properties')
if cp:
warnings.warn("Use `geojson_out` to preserve feature properties",
DeprecationWarning)
band_num = kwargs.get('band_num')
if band_num:
warnings.warn("Use `band` to specify band number", DeprecationWarning)
band = band_num
with Raster(raster, affine, nodata, band) as rast:
features_iter = read_features(vectors, layer)
for _, feat in enumerate(features_iter):
geom = shape(feat['geometry'])
if 'Point' in geom.type:
geom = boxify_points(geom, rast)
geom_bounds = tuple(geom.bounds)
fsrc = rast.read(bounds=geom_bounds)
# rasterized geometry
rv_array = rasterize_geom(geom, like=fsrc, all_touched=all_touched)
# nodata mask
isnodata = (fsrc.array == fsrc.nodata)
# add nan mask (if necessary)
has_nan = (np.issubdtype(fsrc.array.dtype, np.floating)
and np.isnan(fsrc.array.min()))
if has_nan:
isnodata = (isnodata | np.isnan(fsrc.array))
# Mask the source data array
# mask everything that is not a valid value or not within our geom
masked = np.ma.MaskedArray(fsrc.array, mask=(isnodata | ~rv_array))
# If we're on 64 bit platform and the array is an integer type
# make sure we cast to 64 bit to avoid overflow.
# workaround for https://github.com/numpy/numpy/issues/8433
if sysinfo.platform_bits == 64 and \
masked.dtype != np.int64 and \
issubclass(masked.dtype.type, np.integer):
masked = masked.astype(np.int64)
# execute zone_func on masked zone ndarray
if zone_func is not None:
if not callable(zone_func):
raise TypeError(('zone_func must be a callable '
'which accepts function a '
'single `zone_array` arg.'))
zone_func(masked)
if masked.compressed().size == 0:
# nothing here, fill with None and move on
feature_stats = dict([(stat, None) for stat in stats])
if 'count' in stats: # special case, zero makes sense here
feature_stats['count'] = 0
else:
if run_count:
keys, counts = np.unique(masked.compressed(),
return_counts=True)
pixel_count = dict(
zip([np.asscalar(k) for k in keys],
[np.asscalar(c) for c in counts]))
if categorical:
feature_stats = dict(pixel_count)
if category_map:
feature_stats = remap_categories(
category_map, feature_stats)
else:
feature_stats = {}
if 'min' in stats:
feature_stats['min'] = float(masked.min())
if 'max' in stats:
feature_stats['max'] = float(masked.max())
if 'mean' in stats:
feature_stats['mean'] = float(masked.mean())
if 'count' in stats:
feature_stats['count'] = int(masked.count())
# optional
if 'sum' in stats:
feature_stats['sum'] = float(masked.sum())
if 'std' in stats:
feature_stats['std'] = float(masked.std())
if 'median' in stats:
feature_stats['median'] = float(
np.median(masked.compressed()))
if 'majority' in stats:
feature_stats['majority'] = float(
key_assoc_val(pixel_count, max))
if 'minority' in stats:
feature_stats['minority'] = float(
key_assoc_val(pixel_count, min))
if 'unique' in stats:
feature_stats['unique'] = len(list(pixel_count.keys()))
if 'range' in stats:
try:
rmin = feature_stats['min']
except KeyError:
rmin = float(masked.min())
try:
rmax = feature_stats['max']
except KeyError:
rmax = float(masked.max())
feature_stats['range'] = rmax - rmin
for pctile in [
s for s in stats if s.startswith('percentile_')
]:
q = get_percentile(pctile)
pctarr = masked.compressed()
feature_stats[pctile] = np.percentile(pctarr, q)
if 'nodata' in stats or 'nan' in stats:
featmasked = np.ma.MaskedArray(fsrc.array, mask=(~rv_array))
if 'nodata' in stats:
feature_stats['nodata'] = float(
(featmasked == fsrc.nodata).sum())
if 'nan' in stats:
feature_stats['nan'] = float(
np.isnan(featmasked).sum()) if has_nan else 0
if add_stats is not None:
for stat_name, stat_func in add_stats.items():
feature_stats[stat_name] = stat_func(masked)
if raster_out:
feature_stats['mini_raster_array'] = masked
feature_stats['mini_raster_affine'] = fsrc.affine
feature_stats['mini_raster_nodata'] = fsrc.nodata
if prefix is not None:
prefixed_feature_stats = {}
for key, val in feature_stats.items():
newkey = "{}{}".format(prefix, key)
prefixed_feature_stats[newkey] = val
feature_stats = prefixed_feature_stats
if geojson_out:
for key, val in feature_stats.items():
if 'properties' not in feat:
feat['properties'] = {}
feat['properties'][key] = val
yield feat
else:
yield feature_stats
def do_polygon(ctx):
"""polygon workflow"""
pgconn = get_dbconn("postgis")
varname = ctx["v"]
station = ctx["station"][:4]
state = ctx["state"]
phenomena = ctx["phenomena"]
significance = ctx["significance"]
t = ctx["t"]
sdate = ctx["sdate"]
edate = ctx["edate"]
year = ctx["year"]
year2 = ctx["year2"]
# figure out the start and end timestamps
if varname in ["total", "days"]:
sts = sdate
ets = edate
elif varname == "hour":
raise NoDataFound("Sorry, not implemented for polygon summaries.")
elif varname == "yearcount":
sts = datetime.datetime(year, 1, 1).replace(tzinfo=pytz.utc)
ets = datetime.datetime(year, 12, 31, 23, 59).replace(tzinfo=pytz.utc)
else:
sts = datetime.datetime(year, 1, 1).replace(tzinfo=pytz.utc)
ets = datetime.datetime(year2, 12, 31, 23, 59).replace(tzinfo=pytz.utc)
# We need to figure out how to get the warnings either by state or by wfo
if t == "cwa":
(west, south, east, north) = wfo_bounds[station]
else:
(west, south, east, north) = state_bounds[state]
# buffer by 5 degrees so to hopefully get all polys
(west, south) = [x - 2 for x in (west, south)]
(east, north) = [x + 2 for x in (east, north)]
# create grids
griddelta = 0.01
lons = np.arange(west, east, griddelta)
lats = np.arange(south, north, griddelta)
YSZ = len(lats)
XSZ = len(lons)
lons, lats = np.meshgrid(lons, lats)
affine = Affine(griddelta, 0.0, west, 0.0, 0 - griddelta, north)
ones = np.ones((int(YSZ), int(XSZ)))
counts = np.zeros((int(YSZ), int(XSZ)))
wfolimiter = ""
if ctx["t"] == "cwa":
wfolimiter = " wfo = '%s' and " % (station, )
# do arbitrary buffer to prevent segfaults?
df = read_postgis(
"""
SELECT ST_Forcerhr(ST_Buffer(geom, 0.0005)) as geom, issue, expire,
extract(epoch from %s::timestamptz - issue) / 86400. as days
from sbw where """ + wfolimiter + """
phenomena = %s and status = 'NEW' and significance = %s
and ST_Within(geom, ST_GeomFromEWKT('SRID=4326;POLYGON((%s %s, %s %s,
%s %s, %s %s, %s %s))')) and ST_IsValid(geom)
and issue >= %s and issue <= %s ORDER by issue ASC
""",
pgconn,
params=(
ets,
phenomena,
significance,
west,
south,
west,
north,
east,
north,
east,
south,
west,
south,
sts,
ets,
),
geom_col="geom",
index_col=None,
)
if df.empty:
raise NoDataFound("No data found for query.")
zs = zonal_stats(
df["geom"],
ones,
affine=affine,
nodata=-1,
all_touched=True,
raster_out=True,
)
for i, z in enumerate(zs):
aff = z["mini_raster_affine"]
mywest = aff.c
mynorth = aff.f
raster = np.flipud(z["mini_raster_array"])
x0 = int((mywest - west) / griddelta)
y1 = int((mynorth - south) / griddelta)
dy, dx = np.shape(raster)
x1 = x0 + dx
y0 = y1 - dy
if x0 < 0 or x1 >= XSZ or y0 < 0 or y1 >= YSZ:
# print raster.mask.shape, west, x0, x1, XSZ, north, y0, y1, YSZ
continue
if varname == "lastyear":
counts[y0:y1, x0:x1] = np.where(raster.mask, counts[y0:y1, x0:x1],
df.iloc[i]["issue"].year)
elif varname == "days":
counts[y0:y1, x0:x1] = np.where(raster.mask, counts[y0:y1, x0:x1],
df.iloc[i]["days"])
else:
counts[y0:y1, x0:x1] += np.where(raster.mask, 0, 1)
if np.max(counts) == 0:
raise NoDataFound("Sorry, no data found for query!")
# construct the df
ctx["df"] = pd.DataFrame({
"lat": lats.ravel(),
"lon": lons.ravel(),
"val": counts.ravel()
})
minv = df["issue"].min()
maxv = df["issue"].max()
if varname == "lastyear":
ctx["title"] = "Year of Last"
if (maxv.year - minv.year) < 3:
bins = range(int(minv.year) - 4, int(maxv.year) + 2)
else:
bins = range(int(minv.year), int(maxv.year) + 2)
ctx["units"] = "year"
ctx["subtitle"] = (" between %s and %s UTC") % (
sdate.strftime("%d %b %Y %H%M"),
edate.strftime("%d %b %Y %H%M"),
)
elif varname == "days":
ctx["title"] = PDICT2[varname]
bins = np.linspace(max([df["days"].min() - 7, 0]),
df["days"].max() + 7,
12,
dtype="i")
counts = np.where(counts < 0.0001, -1, counts)
ctx["subtitle"] = (" between %s and %s UTC") % (
sdate.strftime("%d %b %Y %H%M"),
edate.strftime("%d %b %Y %H%M"),
)
ctx["units"] = "days"
ctx["extend"] = "neither"
elif varname == "yearcount":
ctx["title"] = "Count for %s" % (year, )
ctx["units"] = "count"
elif varname == "total":
ctx["title"] = "Total"
ctx["subtitle"] = (" between %s and %s UTC") % (
sdate.strftime("%d %b %Y %H%M"),
edate.strftime("%d %b %Y %H%M"),
)
ctx["units"] = "count"
elif varname == "yearavg":
ctx["title"] = ("Yearly Avg: %s and %s") % (
minv.strftime("%d %b %Y"),
maxv.strftime("%d %b %Y"),
)
years = (maxv.year - minv.year) + 1
counts = counts / years
ctx["units"] = "count per year"
maxv = np.max(counts)
if varname not in ["lastyear", "days"]:
if varname == "total":
if maxv < 8:
bins = np.arange(1, 8, 1)
else:
bins = np.linspace(1, maxv + 3, 10, dtype="i")
else:
for delta in [500, 50, 5, 1, 0.5, 0.05]:
bins = np.arange(0, (maxv + 1.0) * 1.05, delta)
if len(bins) > 8:
break
bins[0] = 0.01
ctx["bins"] = bins
ctx["data"] = counts
ctx["lats"] = lats
ctx["lons"] = lons
def main(raster, outputdir, verbose):
""" Convert a raster file (e.g. GeoTIFF) into an HDF5 file.
RASTER: Path to raster file to convert to hdf5.
The HDF5 file has the following datasets:
- Raster: (original image data)
- Latitude: (vector or matrix of pixel latitudes)
- Longitude: (vector or matrix of pixel longitudes)
And the following attributes:
- affine: The affine transformation of the raster
- Various projection information.
"""
if verbose:
print("Opening raster ...")
# Read raster bands directly to Numpy arrays.
# Much of this is from:
# http://gis.stackexchange.com/questions/129847/
# obtain-coordinates-and-corresponding-pixel-values-from-geotiff-using
# -python-gdal
with rasterio.open(os.path.expanduser(raster)) as f:
T0 = f.affine # upper-left pixel corner affine transform
crs = f.crs
p1 = Proj(crs)
I = f.read()
nanvals = f.get_nodatavals()
# Make sure rasterio is always giving us a 3D array
assert (I.ndim == 3)
# This only works on lat-lon projections for now
if not p1.is_latlong():
print("Error: This only works on spherical projections for now (YAGNI"
" you know)...")
exit(1)
if verbose:
print("Extracting coordinate sytem ...")
# Get affine transform for pixel centres
T1 = T0 * Affine.translation(0.5, 0.5)
# Just find lat/lons of axis if there is no rotation/shearing
# https://en.wikipedia.org/wiki/Transformation_matrix#Affine_transformations
if (T1[1] == 0) and (T1[3] == 0):
lons = T1[2] + np.arange(I.shape[2]) * T1[0]
lats = T1[5] + np.arange(I.shape[1]) * T1[4]
# Else, find lat/lons of every pixel!
else:
print("Error: Not yet tested... or even implemented properly!")
exit(1)
# Need to apply affine transformation to all pixel coords
cls, rws = np.meshgrid(np.arange(I.shape[2]), np.arange(I.shape[1]))
# Convert pixel row/column index (from 0) to lat/lon at centre
rc2ll = lambda r, c: (c, r) * T1
# All eastings and northings (there a better way to do this)
lons, lats = np.vectorize(rc2ll, otypes=[np.float, np.float])(rws, cls)
# Permute layers to be more like a standard image, i.e. (band, lon, lat) ->
# (lon, lat, band)
I = (I.transpose([2, 1, 0]))[:, ::-1]
lats = lats[::-1]
# Mask out NaN vals if they exist
if nanvals is not None:
for v in nanvals:
if v is not None:
if verbose:
print("Writing missing values")
I[I == v] = np.nan
# Now write the hdf5
if verbose:
print("Writing HDF5 file ...")
file_stump = os.path.basename(raster).split('.')[-2]
hdf5name = os.path.join(outputdir, file_stump + ".hdf5")
with h5py.File(hdf5name, 'w') as f:
drast = f.create_dataset("Raster", I.shape, dtype=I.dtype, data=I)
drast.attrs['affine'] = T1
for k, v in crs.items():
drast.attrs['k'] = v
f.create_dataset("Latitude", lats.shape, dtype=float, data=lats)
f.create_dataset("Longitude", lons.shape, dtype=float, data=lons)
if verbose:
print("Done!")
def transform_from_latlon(lat, lon):
lat = np.asarray(lat)
lon = np.asarray(lon)
trans = Affine.translation(lon[0], lat[0])
scale = Affine.scale(lon[1] - lon[0], lat[1] - lat[0])
return trans * scale
def from_georef(cls, georef):
tfm = Affine.from_gdal(georef["translateX"], georef["scaleX"],
georef["shearX"], georef["translateY"],
georef["shearY"], georef["scaleY"])
return cls(tfm, proj=georef["spatialReferenceSystemCode"])
def test_read_from_fake_source():
data_source = FakeDataSource()
@contextmanager
def fake_open():
yield data_source
source = mock.Mock()
source.open = fake_open
# one-to-one copy
assert_same_read_results(
source,
dst_shape=data_source.shape,
dst_dtype=data_source.data.dtype,
dst_transform=data_source.transform,
dst_nodata=data_source.nodata,
dst_projection=data_source.crs,
resampling=Resampling.nearest)
# change dtype
assert_same_read_results(
source,
dst_shape=data_source.shape,
dst_dtype='int32',
dst_transform=data_source.transform,
dst_nodata=data_source.nodata,
dst_projection=data_source.crs,
resampling=Resampling.nearest)
# change nodata
assert_same_read_results(
source,
dst_shape=data_source.shape,
dst_dtype='float32',
dst_transform=data_source.transform,
dst_nodata=float('nan'),
dst_projection=data_source.crs,
resampling=Resampling.nearest)
# different offsets/sizes
assert_same_read_results(
source,
dst_shape=(517, 557),
dst_dtype='float32',
dst_transform=data_source.transform * Affine.translation(-200, -200),
dst_nodata=float('nan'),
dst_projection=data_source.crs,
resampling=Resampling.nearest)
assert_same_read_results(
source,
dst_shape=(807, 879),
dst_dtype='float32',
dst_transform=data_source.transform * Affine.translation(200, 200),
dst_nodata=float('nan'),
dst_projection=data_source.crs,
resampling=Resampling.nearest)
assert_same_read_results(
source,
dst_shape=(807, 879),
dst_dtype='float32',
dst_transform=data_source.transform * Affine.translation(1500, -1500),
dst_nodata=float('nan'),
dst_projection=data_source.crs,
resampling=Resampling.nearest)
# flip axis
assert_same_read_results(
source,
dst_shape=(517, 557),
dst_dtype='float32',
dst_transform=data_source.transform * Affine.translation(0, 512) * Affine.scale(1, -1),
dst_nodata=float('nan'),
dst_projection=data_source.crs,
resampling=Resampling.nearest)
assert_same_read_results(
source,
dst_shape=(517, 557),
dst_dtype='float32',
dst_transform=data_source.transform * Affine.translation(512, 0) * Affine.scale(-1, 1),
dst_nodata=float('nan'),
dst_projection=data_source.crs,
resampling=Resampling.nearest)
# scale
assert_same_read_results(
source,
dst_shape=(250, 500),
dst_dtype='float32',
dst_transform=data_source.transform * Affine.scale(2, 4),
dst_nodata=float('nan'),
dst_projection=data_source.crs,
resampling=Resampling.nearest)
assert_same_read_results(
source,
dst_shape=(500, 250),
dst_dtype='float32',
dst_transform=data_source.transform * Affine.scale(4, 2),
dst_nodata=float('nan'),
dst_projection=data_source.crs,
resampling=Resampling.cubic)
assert_same_read_results(
source,
dst_shape=(67, 35),
dst_dtype='float32',
dst_transform=data_source.transform * Affine.scale(16, 8),
dst_nodata=float('nan'),
dst_projection=data_source.crs,
resampling=Resampling.cubic)
assert_same_read_results(
source,
dst_shape=(35, 67),
dst_dtype='float32',
dst_transform=data_source.transform * Affine.translation(27, 35) * Affine.scale(8, 16),
dst_nodata=float('nan'),
dst_projection=data_source.crs,
resampling=Resampling.cubic)
assert_same_read_results(
source,
dst_shape=(35, 67),
dst_dtype='float32',
dst_transform=data_source.transform * Affine.translation(-13, -27) * Affine.scale(8, 16),
dst_nodata=float('nan'),
dst_projection=data_source.crs,
resampling=Resampling.cubic)
# scale + flip
assert_same_read_results(
source,
dst_shape=(35, 67),
dst_dtype='float32',
dst_transform=data_source.transform * Affine.translation(15, 512 + 17) * Affine.scale(8, -16),
dst_nodata=float('nan'),
dst_projection=data_source.crs,
resampling=Resampling.cubic)
assert_same_read_results(
source,
dst_shape=(67, 35),
dst_dtype='float32',
dst_transform=data_source.transform * Affine.translation(512 - 23, -29) * Affine.scale(-16, 8),
dst_nodata=float('nan'),
dst_projection=data_source.crs,
resampling=Resampling.cubic)
class TestRasterDataReading(object):
@pytest.mark.parametrize("dst_nodata", [
np.nan, float("nan"), -999
])
def xtest_failed_data_read(self, make_sample_geotiff, dst_nodata):
sample_geotiff_path, geobox, written_data = make_sample_geotiff(dst_nodata)
src_transform = Affine(25.0, 0.0, 1200000.0,
0.0, -25.0, -4200000.0)
source = RasterFileDataSource(sample_geotiff_path, 1, transform=src_transform)
dest = np.zeros((20, 100))
dst_nodata = -999
dst_projection = geometry.CRS('EPSG:3577')
dst_resampling = Resampling.nearest
# Read exactly the hunk of data that we wrote
dst_transform = Affine(25.0, 0.0, 127327.0,
0.0, -25.0, -417232.0)
read_from_source(source, dest, dst_transform, dst_nodata, dst_projection, dst_resampling)
assert np.all(written_data == dest)
@pytest.mark.parametrize("dst_nodata", [
np.nan, float("nan"), -999
])
def test_read_with_rasterfiledatasource(self, make_sample_geotiff, dst_nodata):
sample_geotiff_path, geobox, written_data = make_sample_geotiff(dst_nodata)
source = RasterFileDataSource(str(sample_geotiff_path), 1)
dest = np.zeros_like(written_data)
dst_transform = geobox.transform
dst_projection = geometry.CRS('EPSG:3577')
dst_resampling = Resampling.nearest
# Read exactly the hunk of data that we wrote
read_from_source(source, dest, dst_transform, dst_nodata, dst_projection, dst_resampling)
assert np.all(written_data == dest)
# Try reading from partially outside of our area
xoff = 50
offset_transform = dst_transform * Affine.translation(xoff, 0)
dest = np.zeros_like(written_data)
read_from_source(source, dest, offset_transform, dst_nodata, dst_projection, dst_resampling)
assert np.all(written_data[:, xoff:] == dest[:, :xoff])
# Try reading from complete outside of our area, should return nodata
xoff = 300
offset_transform = dst_transform * Affine.translation(xoff, 0)
dest = np.zeros_like(written_data)
read_from_source(source, dest, offset_transform, dst_nodata, dst_projection, dst_resampling)
if np.isnan(dst_nodata):
assert np.all(np.isnan(dest))
else:
assert np.all(dst_nodata == dest)
@pytest.mark.parametrize("dst_transform", [
Affine(25.0, 0.0, 1273275.0, 0.0, -25.0, -4172325.0),
Affine(25.0, 0.0, 127327.0, 0.0, -25.0, -417232.0)
])
def test_read_data_from_outside_file_region(self, make_sample_netcdf, dst_transform):
sample_nc, geobox, written_data = make_sample_netcdf
source = RasterFileDataSource(sample_nc, 1)
dest = np.zeros((200, 1000))
dst_nodata = -999
dst_projection = geometry.CRS('EPSG:3577')
dst_resampling = Resampling.nearest
# Read exactly the hunk of data that we wrote
read_from_source(source, dest, dst_transform, dst_nodata, dst_projection, dst_resampling)
assert np.all(dest == -999)
def test_read_with_custom_crs_and_transform(self, example_gdal_path):
with rasterio.open(example_gdal_path) as src:
band = rasterio.band(src, 1)
crs = geometry.CRS('EPSG:3577')
nodata = -999
transform = Affine(25.0, 0.0, 1000000.0,
0.0, -25.0, -900000.0)
# Read all raw data from source file
band_data_source = OverrideBandDataSource(band, nodata, crs, transform)
dest1 = band_data_source.read()
assert dest1.shape
# Attempt to read with the same transform parameters
dest2 = np.full(shape=(4000, 4000), fill_value=nodata, dtype=np.float32)
dst_transform = transform
dst_crs = crs
dst_nodata = nodata
resampling = datacube.storage.storage.RESAMPLING_METHODS['nearest']
band_data_source.reproject(dest2, dst_transform, dst_crs, dst_nodata, resampling)
assert (dest1 == dest2).all()
def test_read_from_file_with_missing_crs(self, no_crs_gdal_path):
"""
We need to be able to read from data files even when GDAL can't automatically gather all the metdata.
The :class:`RasterFileDataSource` is able to override the nodata, CRS and transform attributes if necessary.
"""
crs = geometry.CRS('EPSG:4326')
nodata = -999
transform = Affine(0.01, 0.0, 111.975,
0.0, 0.01, -9.975)
data_source = RasterFileDataSource(no_crs_gdal_path, bandnumber=1, nodata=nodata, crs=crs, transform=transform)
with data_source.open() as src:
dest1 = src.read()
assert dest1.shape == (10, 10)
def mask_image_by_geojson_polygon(geojson_polygon, geoproj, raster):
'''
:param geojson_polygon: the geojson format of a polygon
:param geoproj: the projection coordinate system of the input polygon
:param raster: the raster data after executing the raster = rasterio.open(raster_image_file, 'r')
:return: the data cut out from the raster by the polygon, and its geotransformation
'''
transform = GeomTrans(str(geoproj), str(raster.crs.wkt))
geojson_crs_transformed = transform.transform_points(
geojson_polygon['coordinates'][0])
geometry = shape({
'coordinates': [geojson_crs_transformed],
'type': 'Polygon'
})
bbox = raster.bounds
extent = [[bbox.left, bbox.top], [bbox.left, bbox.bottom],
[bbox.right, bbox.bottom], [bbox.right, bbox.top]]
raster_boundary = shape({'coordinates': [extent], 'type': 'Polygon'})
# if not geometry.intersects(raster_boundary):
# return
if not geometry.within(raster_boundary):
print('the geometry is not within the raster image')
return
# get pixel coordinates of the geometry's bounding box,
ll = raster.index(
*geometry.bounds[0:2]) # lowerleft bounds[0:2] xmin, ymin
ur = raster.index(
*geometry.bounds[2:4]) # upperright bounds[2:4] xmax, ymax
# create an affine transform for the subset data
t = raster.transform
shifted_affine = Affine(t.a, t.b, t.c + ll[1] * t.a, t.d, t.e,
t.f + ur[0] * t.e)
# read the subset of the data into a numpy array
window = ((ur[0], ll[0] + 1), (ll[1], ur[1] + 1))
bands_num = raster.count
multi_bands_data = []
for i in range(bands_num):
# band_name = raster.indexes[i]
data = raster.read(i + 1, window=window)
# rasterize the geometry
mask = rasterio.features.rasterize([(geometry, 0)],
out_shape=data.shape,
transform=shifted_affine,
fill=1,
all_touched=True,
dtype=np.uint8)
# create a masked numpy array
masked_data = np.ma.array(data=data,
mask=mask.astype(bool),
dtype=np.float32)
masked_data.data[
masked_data.mask] = np.nan # raster.profile nodata is 256
out_image = masked_data.data
multi_bands_data.append(out_image)
out_data = np.array(multi_bands_data)
return out_data, shifted_affine
"""Geospatial transforms"""
from __future__ import absolute_import
from __future__ import division
import warnings
from affine import Affine
IDENTITY = Affine.identity()
def tastes_like_gdal(seq):
"""Return True if `seq` matches the GDAL geotransform pattern."""
return seq[2] == seq[4] == 0.0 and seq[1] > 0 and seq[5] < 0
def guard_transform(transform):
"""Return an Affine transformation instance."""
if not isinstance(transform, Affine):
if tastes_like_gdal(transform):
warnings.warn(
"GDAL-style transforms are deprecated and will not "
"be supported in Rasterio 1.0.",
FutureWarning,
stacklevel=2)
transform = Affine.from_gdal(*transform)
else:
transform = Affine(*transform)
return transform
from affine import Affine
import numpy as np
names = [
'africa', 'americas', 'asia', 'europe', 'oceania-east', 'oceania-west'
]
pixel_size = 0.01
xmin = -180
xmax = 180
ymin = -90
ymax = 90
affine = Affine(pixel_size, 0, xmin,
0, -pixel_size, ymax)
shape = (int((ymax-ymin)/pixel_size), int((xmax-xmin)/pixel_size))
roads = np.zeros(shape, dtype='byte')
for n in names:
path = "{0}/{1}".format("/sciclone/aiddata10/REU/geo/raw/groads",
"groads-v1-{0}-shp/gROADS-v1-{0}.shp".format(n))
rv_array, _ = rasterize(path, affine=affine, shape=shape)
roads = roads | rv_array
roads_output_raster_path = "/sciclone/aiddata10/REU/geo/data/rasters/distance_to/groads/binary/groads_binary.tif"
export_raster(roads, affine, roads_output_raster_path)
def transform(self):
bounds = self.metadata.grid_spatial['geo_ref_points']
return Affine(bounds['lr']['x'] - bounds['ul']['x'], 0,
bounds['ul']['x'], 0,
bounds['lr']['y'] - bounds['ul']['y'], bounds['ul']['y'])
def part(
src_dst: Union[DatasetReader, DatasetWriter, WarpedVRT],
bounds: Tuple[float, float, float, float],
height: Optional[int] = None,
width: Optional[int] = None,
padding: int = 0,
dst_crs: Optional[CRS] = None,
bounds_crs: Optional[CRS] = None,
minimum_overlap: Optional[float] = None,
vrt_options: Optional[Dict] = None,
max_size: Optional[int] = None,
**kwargs: Any,
) -> Tuple[numpy.ndarray, numpy.ndarray]:
"""Read part of a dataset.
Args:
src_dst (rasterio.io.DatasetReader or rasterio.io.DatasetWriter or rasterio.vrt.WarpedVRT): Rasterio dataset.
bounds (tuple): Output bounds (left, bottom, right, top). By default the cordinates are considered to be in either the dataset CRS or in the `dst_crs` if set. Use `bounds_crs` to set a specific CRS.
height (int, optional): Output height of the array.
width (int, optional): Output width of the array.
padding (int, optional): Padding to apply to each edge of the tile when retrieving data to assist in reducing resampling artefacts along edges. Defaults to `0`.
dst_crs (rasterio.crs.CRS, optional): Target coordinate reference system.
bounds_crs (rasterio.crs.CRS, optional): Overwrite bounds Coordinate Reference System.
minimum_overlap (float, optional): Minimum % overlap for which to raise an error with dataset not covering enought of the tile.
vrt_options (dict, optional): Options to be passed to the rasterio.warp.WarpedVRT class.
max_size (int, optional): Limit output size array if not widht and height.
kwargs (optional): Additional options to forward to `rio_tiler.reader.read`.
Returns:
tuple: Data (numpy.ndarray) and Mask (numpy.ndarray) values.
"""
if not dst_crs:
dst_crs = src_dst.crs
if max_size and width and height:
warnings.warn(
"'max_size' will be ignored with with 'height' and 'width' set.",
UserWarning,
)
if bounds_crs:
bounds = transform_bounds(bounds_crs, dst_crs, *bounds, densify_pts=21)
if minimum_overlap:
src_bounds = transform_bounds(src_dst.crs,
dst_crs,
*src_dst.bounds,
densify_pts=21)
x_overlap = max(
0,
min(src_bounds[2], bounds[2]) - max(src_bounds[0], bounds[0]))
y_overlap = max(
0,
min(src_bounds[3], bounds[3]) - max(src_bounds[1], bounds[1]))
cover_ratio = (x_overlap * y_overlap) / ((bounds[2] - bounds[0]) *
(bounds[3] - bounds[1]))
if cover_ratio < minimum_overlap:
raise TileOutsideBounds(
"Dataset covers less than {:.0f}% of tile".format(cover_ratio *
100))
vrt_transform, vrt_width, vrt_height = get_vrt_transform(src_dst,
bounds,
height,
width,
dst_crs=dst_crs)
window = windows.Window(col_off=0,
row_off=0,
width=vrt_width,
height=vrt_height)
if max_size and not (width and height):
if max(vrt_width, vrt_height) > max_size:
ratio = vrt_height / vrt_width
if ratio > 1:
height = max_size
width = math.ceil(height / ratio)
else:
width = max_size
height = math.ceil(width * ratio)
out_height = height or vrt_height
out_width = width or vrt_width
if padding > 0 and not is_aligned(src_dst, bounds, out_height, out_width):
vrt_transform = vrt_transform * Affine.translation(-padding, -padding)
orig_vrt_height = vrt_height
orig_vrt_width = vrt_width
vrt_height = vrt_height + 2 * padding
vrt_width = vrt_width + 2 * padding
window = windows.Window(
col_off=padding,
row_off=padding,
width=orig_vrt_width,
height=orig_vrt_height,
)
vrt_options = vrt_options or {}
vrt_options.update({
"crs": dst_crs,
"transform": vrt_transform,
"width": vrt_width,
"height": vrt_height,
})
return read(
src_dst,
out_height,
out_width,
window=window,
vrt_options=vrt_options,
**kwargs,
)
def draw_symbol(part_num, part_ref_prefix, part_footprint, part_manf_num,
pin_data, sort_type, reverse, fuzzy_match):
'''Add a symbol for a part to the library.'''
# Start the part definition with the header.
part_defn = START_DEF.format(name=part_num,
ref=part_ref_prefix,
pin_name_offset=PIN_NAME_OFFSET,
show_pin_number=SHOW_PIN_NUMBER and 'Y'
or 'N',
show_pin_name=SHOW_PIN_NAME and 'Y' or 'N',
num_units=len(pin_data))
# Determine if there are pins across the top of the symbol.
# If so, right-justify the reference and part number so they don't
# run into the top pins. If not, stick with left-justification.
horiz_just = 'L'
horiz_offset = PIN_LENGTH
for unit in list(pin_data.values()):
if 'top' in list(unit.keys()):
horiz_just = 'R'
horiz_offset = PIN_LENGTH - 50
break
# Create the field that stores the part reference.
if part_ref_prefix:
part_defn += REF_FIELD.format(ref_prefix=part_ref_prefix,
x=XO + horiz_offset,
y=YO + REF_Y_OFFSET,
horiz_just=horiz_just,
ref_size=REF_SIZE)
# Create the field that stores the part number.
if part_num:
part_defn += PART_FIELD.format(part_num=part_num,
x=XO + horiz_offset,
y=YO + PART_NUM_Y_OFFSET,
horiz_just=horiz_just,
ref_size=PART_NUM_SIZE)
# Create the field that stores the part footprint.
if part_footprint:
part_defn += FOOTPRINT_FIELD.format(footprint=part_footprint,
x=XO + horiz_offset,
y=YO + PART_FOOTPRINT_Y_OFFSET,
horiz_just=horiz_just,
ref_size=PART_FOOTPRINT_SIZE)
# Create the field that stores the manufacturer part number.
if part_manf_num:
part_defn += MPN_FIELD.format(manf_num=part_manf_num,
x=XO + horiz_offset,
y=YO + PART_MPN_Y_OFFSET,
horiz_just=horiz_just,
ref_size=PART_MPN_SIZE)
# Start the section of the part definition that holds the part's units.
part_defn += START_DRAW
# Get a reference to the sort-key generation function for pins.
pin_key_func = getattr(THIS_MODULE, '{}_key'.format(sort_type))
# This is the sort-key generation function for unit names.
unit_key_func = lambda x: zero_pad_nums(x[0])
# Now create the units that make up the part. Unit numbers go from 1
# up to the number of units in the part. The units are sorted by their
# names before assigning unit numbers.
for unit_num, unit in enumerate(
[p[1] for p in sorted(pin_data.items(), key=unit_key_func)], 1):
# The indices of the X and Y coordinates in a list of point coords.
X = 0
Y = 1
# Initialize data structures that store info for each side of a schematic symbol unit.
all_sides = ['left', 'right', 'top', 'bottom']
bbox = {side: [(XO, YO), (XO, YO)] for side in all_sides}
box_pt = {
side: [XO + PIN_LENGTH, YO + PIN_SPACING]
for side in all_sides
}
anchor_pt = {
side: [XO + PIN_LENGTH, YO + PIN_SPACING]
for side in all_sides
}
transform = {}
# Annotate the pins for each side of the symbol.
for side_pins in list(unit.values()):
annotate_pins(list(side_pins.items()))
# Determine the actual bounding box for each side.
bbox = {}
for side, side_pins in list(unit.items()):
bbox[side] = pins_bbox(list(side_pins.items()))
# Adjust the sizes of the bboxes to make the unit look more symmetrical.
balance_bboxes(bbox)
# Determine some important points for each side of pins.
for side in unit:
#
# C B-------A
# | |
# ------| name1 |
# | |
# ------| name2 |
#
# A = anchor point = upper-right corner of bounding box.
# B = box point = upper-left corner of bounding box + pin length.
# C = upper-left corner of bounding box.
anchor_pt[side] = [
max(bbox[side][0][X], bbox[side][1][X]),
max(bbox[side][0][Y], bbox[side][1][Y])
]
box_pt[side] = [
min(bbox[side][0][X], bbox[side][1][X]) + PIN_LENGTH,
max(bbox[side][0][Y], bbox[side][1][Y])
]
# AL = left-side anchor point.
# AB = bottom-side anchor point.
# AR = right-side anchor point.
# AT = top-side anchor-point.
# +-------------+
# | |
# | TOP |
# | |
# +------AL------------AT
# | |
# | | +---------+
# | | | |
# | L | | |
# | E | | R |
# | F | | I |
# | T | | G |
# | | | H |
# | | | T |
# | | | |
# +------AB-------+ AR--------+
# | BOTTOM |
# +--------+
#
# Create zero-sized bounding boxes for any sides of the unit without pins.
# This makes it simpler to do the width/height calculation that follows.
for side in all_sides:
if side not in bbox:
bbox[side] = [(XO, YO), (XO, YO)]
# This is the width and height of the box in the middle of the pins on each side.
box_width = max(abs(bbox['top'][0][Y] - bbox['top'][1][Y]),
abs(bbox['bottom'][0][Y] - bbox['bottom'][1][Y]))
box_height = max(abs(bbox['left'][0][Y] - bbox['left'][1][Y]),
abs(bbox['right'][0][Y] - bbox['right'][1][Y]))
for side in all_sides:
# Each side of pins starts off with the orientation of a left-hand side of pins.
# Transformation matrix starts by rotating the side of pins.
transform[side] = Affine.rotation(ROTATION[side])
# Now rotate the anchor point to see where it goes.
rot_anchor_pt = transform[side] * anchor_pt[side]
# Translate the rotated anchor point to coincide with the AL anchor point.
translate_x = anchor_pt['left'][X] - rot_anchor_pt[X]
translate_y = anchor_pt['left'][Y] - rot_anchor_pt[Y]
# Make additional translation to bring the AL point to the correct position.
if side == 'right':
# Translate AL to AR.
translate_x += box_width
translate_y -= box_height
elif side == 'bottom':
# Translate AL to AB
translate_y -= box_height
elif side == 'top':
# Translate AL to AT
translate_x += box_width
# Create the complete transformation matrix = rotation followed by translation.
transform[side] = Affine.translation(translate_x,
translate_y) * transform[side]
# Also translate the point on each side that defines the box around the symbol.
box_pt[side] = transform[side] * box_pt[side]
# Draw the transformed pins for each side of the symbol.
for side, side_pins in list(unit.items()):
# If the pins are ordered by their row in the spreadsheet or by their name,
# then reverse their order on the right and top sides so they go from top-to-bottom
# on the right side and left-to-right on the top side instead of the opposite
# as happens with counter-clockwise pin-number ordering.
side_reverse = reverse
if sort_type in ['name', 'row'] and side in ['right', 'top']:
side_reverse = not reverse
# Sort the pins for the desired order: row-wise, numeric (pin #), alphabetical (pin name).
sorted_side_pins = sorted(list(side_pins.items()),
key=pin_key_func,
reverse=side_reverse)
# Draw the transformed pins for this side of the symbol.
part_defn += draw_pins(unit_num, sorted_side_pins, bbox[side],
transform[side], fuzzy_match)
# Create the box around the unit's pins.
part_defn += BOX.format(x0=int(box_pt['left'][X]),
y0=int(box_pt['top'][Y]),
x1=int(box_pt['right'][X]),
y1=int(box_pt['bottom'][Y]),
unit_num=unit_num,
line_width=BOX_LINE_WIDTH,
fill=FILLS[FILL])
# Close the section that holds the part's units.
part_defn += END_DRAW
# Close the part definition.
part_defn += END_DEF
# Return complete part symbol definition.
return part_defn
def calc_transform(src,
dst_crs=None,
resolution=None,
dimensions=None,
src_bounds=None,
dst_bounds=None,
target_aligned_pixels=False):
"""Output dimensions and transform for a reprojection.
Parameters
------------
src: rasterio.io.DatasetReader
Data source.
dst_crs: rasterio.crs.CRS, optional
Target coordinate reference system.
resolution: tuple (x resolution, y resolution) or float, optional
Target resolution, in units of target coordinate reference
system.
dimensions: tuple (width, height), optional
Output file size in pixels and lines.
src_bounds: tuple (xmin, ymin, xmax, ymax), optional
Georeferenced extent of output file from source bounds
(in source georeferenced units).
dst_bounds: tuple (xmin, ymin, xmax, ymax), optional
Georeferenced extent of output file from destination bounds
(in destination georeferenced units).
target_aligned_pixels: bool, optional
Align the output bounds based on the resolution.
Default is `False`.
Returns
-------
dst_crs: rasterio.crs.CRS
Output crs
transform: Affine
Output affine transformation matrix
width, height: int
Output dimensions
"""
if resolution is not None:
if isinstance(resolution, (float, int)):
resolution = (float(resolution), float(resolution))
if target_aligned_pixels:
if not resolution:
raise ValueError(
'target_aligned_pixels cannot be used without resolution')
if src_bounds or dst_bounds:
raise ValueError(
'target_aligned_pixels cannot be used with src_bounds or dst_bounds'
)
elif dimensions:
invalid_combos = (dst_bounds, resolution)
if any(p for p in invalid_combos if p is not None):
raise ValueError(
'dimensions cannot be used with dst_bounds or resolution')
if src_bounds and dst_bounds:
raise ValueError(
'src_bounds and dst_bounds may not be specified simultaneously')
if dst_crs is not None:
if dimensions:
# Calculate resolution appropriate for dimensions
# in target.
dst_width, dst_height = dimensions
bounds = src_bounds or src.bounds
xmin, ymin, xmax, ymax = transform_bounds(src.crs, dst_crs,
*bounds)
dst_transform = Affine((xmax - xmin) / float(dst_width), 0, xmin,
0, (ymin - ymax) / float(dst_height), ymax)
elif src_bounds or dst_bounds:
if not resolution:
raise ValueError(
'resolution is required when using src_bounds or dst_bounds'
)
if src_bounds:
xmin, ymin, xmax, ymax = transform_bounds(
src.crs, dst_crs, *src_bounds)
else:
xmin, ymin, xmax, ymax = dst_bounds
dst_transform = Affine(resolution[0], 0, xmin, 0, -resolution[1],
ymax)
dst_width = max(int(ceil((xmax - xmin) / resolution[0])), 1)
dst_height = max(int(ceil((ymax - ymin) / resolution[1])), 1)
else:
if src.transform.is_identity and src.gcps:
src_crs = src.gcps[1]
kwargs = {'gcps': src.gcps[0]}
else:
src_crs = src.crs
kwargs = src.bounds._asdict()
dst_transform, dst_width, dst_height = calcdt(
src_crs,
dst_crs,
src.width,
src.height,
resolution=resolution,
**kwargs)
elif dimensions:
# Same projection, different dimensions, calculate resolution.
dst_crs = src.crs
dst_width, dst_height = dimensions
l, b, r, t = src_bounds or src.bounds
dst_transform = Affine((r - l) / float(dst_width), 0, l, 0,
(b - t) / float(dst_height), t)
elif src_bounds or dst_bounds:
# Same projection, different dimensions and possibly
# different resolution.
if not resolution:
resolution = (src.transform.a, -src.transform.e)
dst_crs = src.crs
xmin, ymin, xmax, ymax = (src_bounds or dst_bounds)
dst_transform = Affine(resolution[0], 0, xmin, 0, -resolution[1], ymax)
dst_width = max(int(ceil((xmax - xmin) / resolution[0])), 1)
dst_height = max(int(ceil((ymax - ymin) / resolution[1])), 1)
elif resolution:
# Same projection, different resolution.
dst_crs = src.crs
l, b, r, t = src.bounds
dst_transform = Affine(resolution[0], 0, l, 0, -resolution[1], t)
dst_width = max(int(ceil((r - l) / resolution[0])), 1)
dst_height = max(int(ceil((t - b) / resolution[1])), 1)
else:
dst_crs = src.crs
dst_transform = src.transform
dst_width = src.width
dst_height = src.height
if target_aligned_pixels:
dst_transform, dst_width, dst_height = aligned_target(
dst_transform, dst_width, dst_height, resolution)
return dst_crs, dst_transform, dst_width, dst_height
def write_gtiff(fname,
pix,
crs='epsg:3857',
resolution=(10, -10),
offset=(0.0, 0.0),
nodata=None,
overwrite=False,
blocksize=None,
gbox=None,
**extra_rio_opts):
""" Write ndarray to GeoTiff file.
Geospatial info can be supplied either via
- resolution, offset, crs
or
- gbox (takes precedence if supplied)
"""
# pylint: disable=too-many-locals
from affine import Affine
import rasterio
from pathlib import Path
if pix.ndim == 2:
h, w = pix.shape
nbands = 1
band = 1
elif pix.ndim == 3:
nbands, h, w = pix.shape
band = tuple(i for i in range(1, nbands+1))
else:
raise ValueError('Need 2d or 3d ndarray on input')
if not isinstance(fname, Path):
fname = Path(fname)
if fname.exists():
if overwrite:
fname.unlink()
else:
raise IOError("File exists")
if gbox is not None:
assert gbox.shape == (h, w)
A = gbox.transform
crs = str(gbox.crs)
else:
sx, sy = resolution
tx, ty = offset
A = Affine(sx, 0, tx,
0, sy, ty)
rio_opts = dict(width=w,
height=h,
count=nbands,
dtype=pix.dtype.name,
crs=crs,
transform=A,
predictor=2,
compress='DEFLATE')
if blocksize is not None:
rio_opts.update(tiled=True,
blockxsize=min(blocksize, w),
blockysize=min(blocksize, h))
if nodata is not None:
rio_opts.update(nodata=nodata)
rio_opts.update(extra_rio_opts)
with rasterio.open(str(fname), 'w', driver='GTiff', **rio_opts) as dst:
dst.write(pix, band)
meta = dst.meta
meta['gbox'] = gbox if gbox is not None else rio_geobox(meta)
meta['path'] = fname
return SimpleNamespace(**meta)
rf_min, '%s/%s_%s_%s_rf_iterative_min.pkl' %
(path2alg, country_code, version, source))
joblib.dump(
rf_max, '%s/%s_%s_%s_rf_iterative_max.pkl' %
(path2alg, country_code, version, source))
"""
#===============================================================================
PART C: PRODUCE FULL POTENTIAL BIOMASS MAPS AND SAVE TO NETCDF
#-------------------------------------------------------------------------------
"""
print("PART C: Produce full maps and uncertainty bounds")
# convert training set and AGBpot to xdarray for easy plotting and export to
# netcdf
# first deal with metadata and coordinates
tr = agb.attrs['transform']
transform = Affine(tr[0], tr[1], tr[2], tr[3], tr[4], tr[5])
nx, ny = agb.sizes['x'], agb.sizes['y']
col, row = np.meshgrid(np.arange(nx) + 0.5, np.arange(ny) + 0.5)
lon, lat = transform * (col, row)
coords = {
'lat': (['lat'], lat[:, 0], {
'units': 'degrees_north',
'long_name': 'latitude'
}),
'lon': (['lon'], lon[0, :], {
'units': 'degrees_east',
'long_name': 'longitude'
})
}
def default_ctx(self):
"""
Return an AOI GeoContext for loading this Scene's original, unwarped data.
These defaults are used:
* resolution: resolution determined from the Scene's ``geotrans``
* crs: native CRS of the Scene (often, a UTM CRS)
* bounds: bounds determined from the Scene's ``geotrans`` and ``raster_size``
* bounds_crs: native CRS of the Scene
* align_pixels: False, to prevent interpolation snapping pixels to a new grid
* geometry: None
.. note::
Using this GeoContext will only return original, unwarped data
if the Scene is axis-aligned ("north-up") within the CRS.
If its ``geotrans`` applies a rotation, a warning will be raised.
In that case, use `Raster.ndarray` or `Raster.raster` to retrieve
original data. (The GeoContext paradigm requires bounds for consistentcy,
which are inherently axis-aligned.)
Returns
-------
ctx: AOI
"""
resolution = None
bounds = None
bounds_crs = None
crs = self.properties.get('crs')
geotrans = self.properties.get('geotrans')
if geotrans is not None:
geotrans = Affine.from_gdal(*geotrans)
if not geotrans.is_rectilinear:
# NOTE: this may still be an insufficient check for some CRSs, i.e. polar stereographic?
warnings.warn(
"The GeoContext will *not* return this Scene's original data, "
"since it's rotated compared to the grid of the CRS. "
"The array will be 'north-up', with the data rotated within it, "
"and extra empty pixels padded around the side(s). "
"To get the original, unrotated data, you must use the Raster API: "
"`dl.raster.ndarray(scene.properties.id, ...)`.")
scaling1, scaling2 = geotrans._scaling
if scaling1 == scaling2:
resolution = scaling1
else:
# if pixels aren't square (unlikely), we won't just pick a resolution---user has to figure that out.
warnings.warn(
"Scene has non-square pixels, so no single resolution can be assigned. "
"Use `shape` instead for more predictable results.")
raster_size = self.properties.get('raster_size')
if raster_size is not None:
cols, rows = raster_size
# upper-left, upper-right, lower-left, lower-right in pixel coordinates
pixel_corners = [(0, 0), (cols, 0), (0, rows), (cols, rows)]
geo_corners = [geotrans * corner for corner in pixel_corners]
xs, ys = zip(*geo_corners)
bounds = (min(xs), min(ys), max(xs), max(ys))
bounds_crs = crs
return geocontext.AOI(
geometry=None,
resolution=resolution,
bounds=bounds,
bounds_crs=bounds_crs,
crs=crs,
align_pixels=False,
)
from common_for_tests import make_test_raster
some_array = np.array([[0, 1, 2], [3, 4, 5]], dtype=np.uint8)
some_mask = np.array([[False, False, False], [False, False, True]],
dtype=np.bool)
some_image_2d = np.ma.array(some_array, mask=some_mask)
some_image_2d_alt = np.ma.array(np.array([[0, 1, 2], [3, 4, 99]],
dtype=np.uint8),
mask=some_mask)
some_image_3d = np.ma.array(some_array[np.newaxis, :, :],
mask=some_mask[np.newaxis, :, :])
some_image_3d_multiband = np.ma.array(
np.array([some_array, some_array, some_array]),
mask=np.array([some_mask, some_mask, some_mask]))
raster_origin = Point(2, 3)
some_affine = Affine.translation(raster_origin.x, raster_origin.y)
some_crs = CRS({'init': 'epsg:32620'})
some_raster = GeoRaster2(some_image_2d,
affine=some_affine,
crs=some_crs,
band_names=['r'])
some_raster_alt = GeoRaster2(some_image_2d_alt,
affine=some_affine,
crs=some_crs,
band_names=['r'])
some_raster_multiband = GeoRaster2(some_image_3d_multiband,
band_names=['r', 'g', 'b'],
affine=some_affine,
crs=some_crs)
default_factors = [2, 4, 8, 16]
def footprint_mask(df,
out_file=None,
reference_im=None,
geom_col='geometry',
do_transform=None,
affine_obj=None,
shape=(900, 900),
out_type='int',
burn_value=255,
burn_field=None):
"""Convert a dataframe of geometries to a pixel mask.
Arguments
---------
df : :class:`pandas.DataFrame` or :class:`geopandas.GeoDataFrame`
A :class:`pandas.DataFrame` or :class:`geopandas.GeoDataFrame` instance
with a column containing geometries (identified by `geom_col`). If the
geometries in `df` are not in pixel coordinates, then `affine` or
`reference_im` must be passed to provide the transformation to convert.
out_file : str, optional
Path to an image file to save the output to. Must be compatible with
:class:`rasterio.DatasetReader`. If provided, a `reference_im` must be
provided (for metadata purposes).
reference_im : :class:`rasterio.DatasetReader` or `str`, optional
An image to extract necessary coordinate information from: the
affine transformation matrix, the image extent, etc. If provided,
`affine_obj` and `shape` are ignored.
geom_col : str, optional
The column containing geometries in `df`. Defaults to ``"geometry"``.
do_transform : bool, optional
Should the values in `df` be transformed from geospatial coordinates
to pixel coordinates? Defaults to ``None``, in which case the function
attempts to infer whether or not a transformation is required based on
the presence or absence of a CRS in `df`. If ``True``, either
`reference_im` or `affine_obj` must be provided as a source for the
the required affine transformation matrix.
affine_obj : `list` or :class:`affine.Affine`, optional
Affine transformation to use to convert from geo coordinates to pixel
space. Only provide this argument if `df` is a
:class:`geopandas.GeoDataFrame` with coordinates in a georeferenced
coordinate space. Ignored if `reference_im` is provided.
shape : tuple, optional
An ``(x_size, y_size)`` tuple defining the pixel extent of the output
mask. Ignored if `reference_im` is provided.
out_type : 'float' or 'int'
burn_value : `int` or `float`, optional
The value to use for labeling objects in the mask. Defaults to 255 (the
max value for ``uint8`` arrays). The mask array will be set to the same
dtype as `burn_value`. Ignored if `burn_field` is provided.
burn_field : str, optional
Name of a column in `df` that provides values for `burn_value` for each
independent object. If provided, `burn_value` is ignored.
Returns
-------
mask : :class:`numpy.array`
A pixel mask with 0s for non-object pixels and `burn_value` at object
pixels. `mask` dtype will coincide with `burn_value`.
"""
# start with required checks and pre-population of values
if out_file and not reference_im:
raise ValueError(
'If saving output to file, `reference_im` must be provided.')
df = _check_df_load(df)
if len(df) == 0 and not out_file:
return np.zeros(shape=shape, dtype='uint8')
if do_transform is None:
# determine whether or not transform should be done
do_transform = _check_do_transform(df, reference_im, affine_obj)
df[geom_col] = df[geom_col].apply(_check_geom) # load in geoms if wkt
if not do_transform:
affine_obj = Affine(1, 0, 0, 0, 1, 0) # identity transform
if reference_im:
reference_im = _check_rasterio_im_load(reference_im)
shape = reference_im.shape
if do_transform:
affine_obj = reference_im.transform
# extract geometries and pair them with burn values
if burn_field:
if out_type == 'int':
feature_list = list(
zip(df[geom_col], df[burn_field].astype('uint8')))
else:
feature_list = list(
zip(df[geom_col], df[burn_field].astype('float32')))
else:
feature_list = list(zip(df[geom_col], [burn_value] * len(df)))
if len(df) > 0:
output_arr = features.rasterize(shapes=feature_list,
out_shape=shape,
transform=affine_obj)
else:
output_arr = np.zeros(shape=shape, dtype='uint8')
if out_file:
meta = reference_im.meta.copy()
meta.update(count=1)
if out_type == 'int':
meta.update(dtype='uint8')
meta.update(nodata=0)
with rasterio.open(out_file, 'w', **meta) as dst:
dst.write(output_arr, indexes=1)
return output_arr
def transform_to_coords(transform,
bbox=None,
width=None,
height=None,
center=True):
""" Return the coordinates for a given transform
This function needs to know how many pixels are in the raster,
so either a :py:class:`rasterio.coords.BoundingBox` or
height and width arguments must be supplied.
Parameters
----------
transform : affine.Affine
Affine transform
bbox : BoundingBox
Return the coordinates for this transform that are contained inside
the provided bounds (left, bottom, right, up). Coordinates will
be aligned/spaced according to the transform and will not necessarily
match the upper-left of this ``bbox``
height : int
Number of pixels tall
width : int
Number of pixels wide
center : bool, optional
Return coordinates for the center of each pixel
Returns
-------
y, x : tuple[np.ndarray, np.ndarray]
Y/X coordinate pairs (e.g., latitude/longitude)
"""
# TODO Guard type of transform and bbox using convert func
# transform = convert.to_transform(transform)
if bbox is not None:
# TODO Guard bbox
# bbox = convert.to_bbox(bbox)
# Find nearest y/x for bbox given posting from transform
off_y = math.floor((bbox.top - transform.f) / transform.e)
off_x = math.floor((bbox.left - transform.c) / transform.a)
# Determine (potentially) new upper-left for bbox
top = transform.f + off_y * transform.e
left = transform.c + off_x * transform.a
# Calculate size needed to cover bounds
height = int(math.ceil(abs((top - bbox.bottom) / transform.e)))
width = int(math.ceil(abs((bbox.right - left) / transform.a)))
# Move the transform to this location
transform = transform * Affine.translation(off_x, off_y)
elif not (height and width):
raise ValueError("Must provide either a BoundingBox (`bbox`) "
"or the dimensions (`height` and `width`) "
"of the raster grid.")
offset = 0.5 if center else 0.0
x, _ = (np.arange(width) + offset, np.zeros(width) + offset) * transform
_, y = (np.zeros(height) + offset, np.arange(height) + offset) * transform
return (y, x)
def test_copy_with():
new_affine = Affine.translation(42, 42)
assert some_raster.copy_with(affine=new_affine).affine == new_affine
def calculate_default_transform(src_crs,
dst_crs,
width,
height,
left,
bottom,
right,
top,
resolution=None):
"""Calculate parameters for reproject function.
Transforms bounds to destination coordinate system, calculates resolution
if not provided, and returns destination transform and dimensions.
Intended to be used to calculate parameters for reproject function.
Destination transform is anchored from the left, top coordinate.
Destination width and height (and resolution if not provided), are
calculated using GDAL's method for suggest warp output.
Parameters
----------
src_crs: CRS or dict
Source coordinate reference system, in rasterio dict format.
Example: CRS({'init': 'EPSG:4326'})
dst_crs: CRS or dict
Target coordinate reference system.
width: int
Source raster width.
height: int
Source raster height.
left, bottom, right, top: float
Bounding coordinates in src_crs, from the bounds property of a raster.
resolution: tuple (x resolution, y resolution) or float, optional
Target resolution, in units of target coordinate reference system.
Returns
-------
tuple of destination affine transform, width, and height
Note
----
Should be called within a rasterio.Env() context
Some behavior of this function is determined by the
CHECK_WITH_INVERT_PROJ environment variable
YES: constrain output raster to extents that can be inverted
avoids visual artifacts and coordinate discontinuties.
NO: reproject coordinates beyond valid bound limits
"""
dst_affine, dst_width, dst_height = _calculate_default_transform(
src_crs, dst_crs, width, height, left, bottom, right, top)
# If resolution is specified, Keep upper-left anchored
# adjust the transform resolutions
# adjust the width/height by the ratio of estimated:specified res (ceil'd)
if resolution:
# resolutions argument into tuple
try:
res = (float(resolution), float(resolution))
except TypeError:
res = (resolution[0], resolution[0]) \
if len(resolution) == 1 else resolution[0:2]
# Assume yres is provided as positive,
# needs to be negative for north-up affine
xres = res[0]
yres = -res[1]
xratio = dst_affine.a / xres
yratio = dst_affine.e / yres
dst_affine = Affine(xres, dst_affine.b, dst_affine.c, dst_affine.d,
yres, dst_affine.f)
dst_width = ceil(dst_width * xratio)
dst_height = ceil(dst_height * yratio)
return dst_affine, dst_width, dst_height
def test_area():
scale = 2
raster = some_raster.copy_with(affine=some_raster.affine *
Affine.scale(scale))
expected = raster.width * raster.height * (scale**2)
assert pytest.approx(expected, .01) == raster.area()
def test_resolution():
raster = some_raster.deepcopy_with(affine=Affine.scale(2, 3))
assert raster.resolution() == np.sqrt(2 * 3)
def _get_raster_tile(cls, path: str, *,
reprojection_method: str,
resampling_method: str,
tile_bounds: Tuple[float, float, float, float] = None,
tile_size: Tuple[int, int] = (256, 256),
preserve_values: bool = False) -> np.ma.MaskedArray:
"""Load a raster dataset from a file through rasterio.
Heavily inspired by mapbox/rio-tiler
"""
import rasterio
from rasterio import transform, windows, warp
from rasterio.vrt import WarpedVRT
from affine import Affine
dst_bounds: Tuple[float, float, float, float]
if preserve_values:
reproject_enum = resampling_enum = cls._get_resampling_enum('nearest')
else:
reproject_enum = cls._get_resampling_enum(reprojection_method)
resampling_enum = cls._get_resampling_enum(resampling_method)
with contextlib.ExitStack() as es:
es.enter_context(rasterio.Env(**cls._RIO_ENV_KEYS))
try:
with trace('open_dataset'):
src = es.enter_context(rasterio.open(path))
except OSError:
raise IOError('error while reading file {}'.format(path))
# compute buonds in target CRS
dst_bounds = warp.transform_bounds(src.crs, cls._TARGET_CRS, *src.bounds)
if tile_bounds is None:
tile_bounds = dst_bounds
# prevent loads of very sparse data
cover_ratio = (
(dst_bounds[2] - dst_bounds[0]) / (tile_bounds[2] - tile_bounds[0])
* (dst_bounds[3] - dst_bounds[1]) / (tile_bounds[3] - tile_bounds[1])
)
if cover_ratio < 0.01:
raise exceptions.TileOutOfBoundsError('dataset covers less than 1% of tile')
# compute suggested resolution in target CRS
dst_transform, _, _ = warp.calculate_default_transform(
src.crs, cls._TARGET_CRS, src.width, src.height, *src.bounds
)
dst_res = (abs(dst_transform.a), abs(dst_transform.e))
# make sure VRT resolves the entire tile
tile_transform = transform.from_bounds(*tile_bounds, *tile_size)
tile_res = (abs(tile_transform.a), abs(tile_transform.e))
if tile_res[0] < dst_res[0] or tile_res[1] < dst_res[1]:
dst_res = tile_res
resampling_enum = cls._get_resampling_enum('nearest')
# pad tile bounds to prevent interpolation artefacts
num_pad_pixels = 2
# compute tile VRT shape and transform
dst_width = max(1, round((tile_bounds[2] - tile_bounds[0]) / dst_res[0]))
dst_height = max(1, round((tile_bounds[3] - tile_bounds[1]) / dst_res[1]))
vrt_transform = (
transform.from_bounds(*tile_bounds, width=dst_width, height=dst_height)
* Affine.translation(-num_pad_pixels, -num_pad_pixels)
)
vrt_height, vrt_width = dst_height + 2 * num_pad_pixels, dst_width + 2 * num_pad_pixels
# remove padding in output
out_window = windows.Window(
col_off=num_pad_pixels, row_off=num_pad_pixels, width=dst_width, height=dst_height
)
# construct VRT
vrt = es.enter_context(
WarpedVRT(
src, crs=cls._TARGET_CRS, resampling=reproject_enum,
transform=vrt_transform, width=vrt_width, height=vrt_height,
add_alpha=not cls._has_alpha_band(src)
)
)
# read data
with warnings.catch_warnings(), trace('read_from_vrt'):
warnings.filterwarnings('ignore', message='invalid value encountered.*')
tile_data = vrt.read(
1, resampling=resampling_enum, window=out_window, out_shape=tile_size
)
# assemble alpha mask
mask_idx = vrt.count
mask = vrt.read(mask_idx, window=out_window, out_shape=tile_size) == 0
if src.nodata is not None:
mask |= tile_data == src.nodata
return np.ma.masked_array(tile_data, mask=mask)
def calculate_default_transform(
src_crs, dst_crs, width, height, left=None, bottom=None, right=None,
top=None, gcps=None, resolution=None, dst_width=None, dst_height=None):
"""Output dimensions and transform for a reprojection.
Source and destination coordinate reference systems and output
width and height are the first four, required, parameters. Source
georeferencing can be specified using either ground control points
(gcps) or spatial bounds (left, bottom, right, top). These two
forms of georeferencing are mutually exclusive.
The destination transform is anchored at the left, top coordinate.
Destination width and height (and resolution if not provided), are
calculated using GDAL's method for suggest warp output.
Parameters
----------
src_crs: CRS or dict
Source coordinate reference system, in rasterio dict format.
Example: CRS({'init': 'EPSG:4326'})
dst_crs: CRS or dict
Target coordinate reference system.
width, height: int
Source raster width and height.
left, bottom, right, top: float, optional
Bounding coordinates in src_crs, from the bounds property of a
raster. Required unless using gcps.
gcps: sequence of GroundControlPoint, optional
Instead of a bounding box for the source, a sequence of ground
control points may be provided.
resolution: tuple (x resolution, y resolution) or float, optional
Target resolution, in units of target coordinate reference
system.
dst_width, dst_height: int, optional
Output file size in pixels and lines. Cannot be used together
with resolution.
Returns
-------
transform: Affine
Output affine transformation matrix
width, height: int
Output dimensions
Notes
-----
Some behavior of this function is determined by the
CHECK_WITH_INVERT_PROJ environment variable:
YES: constrain output raster to extents that can be inverted
avoids visual artifacts and coordinate discontinuties.
NO: reproject coordinates beyond valid bound limits
"""
if any(x is not None for x in (left, bottom, right, top)) and gcps:
raise ValueError("Bounding values and ground control points may not"
"be used together.")
if any(x is None for x in (left, bottom, right, top)) and not gcps:
raise ValueError("Either four bounding values or ground control points"
"must be specified")
if (dst_width is None) != (dst_height is None):
raise ValueError("Either dst_width and dst_height must be specified "
"or none of them.")
if all(x is not None for x in (dst_width, dst_height)):
dimensions = (dst_width, dst_height)
else:
dimensions = None
if resolution and dimensions:
raise ValueError("Resolution cannot be used with dst_width and dst_height.")
dst_affine, dst_width, dst_height = _calculate_default_transform(
src_crs, dst_crs, width, height, left, bottom, right, top, gcps)
# If resolution is specified, Keep upper-left anchored
# adjust the transform resolutions
# adjust the width/height by the ratio of estimated:specified res (ceil'd)
if resolution:
# resolutions argument into tuple
try:
res = (float(resolution), float(resolution))
except TypeError:
res = (resolution[0], resolution[0]) \
if len(resolution) == 1 else resolution[0:2]
# Assume yres is provided as positive,
# needs to be negative for north-up affine
xres = res[0]
yres = -res[1]
xratio = dst_affine.a / xres
yratio = dst_affine.e / yres
dst_affine = Affine(xres, dst_affine.b, dst_affine.c,
dst_affine.d, yres, dst_affine.f)
dst_width = ceil(dst_width * xratio)
dst_height = ceil(dst_height * yratio)
if dimensions:
xratio = dst_width / dimensions[0]
yratio = dst_height / dimensions[1]
dst_width = dimensions[0]
dst_height = dimensions[1]
dst_affine = Affine(dst_affine.a * xratio, dst_affine.b, dst_affine.c,
dst_affine.d, dst_affine.e * yratio, dst_affine.f)
return dst_affine, dst_width, dst_height
def mask_image_by_geometry(geomjson, geomproj, raster, tag, name):
print('the %s geometry' % name)
transform = GeomTrans(str(geomproj), str(raster.crs.wkt))
geojson_crs_transformed = transform.transform_points(
geomjson['coordinates'][0])
geometry = shape({
'coordinates': [geojson_crs_transformed],
'type': 'Polygon'
})
bbox = raster.bounds
extent = [[bbox.left, bbox.top], [bbox.left, bbox.bottom],
[bbox.right, bbox.bottom], [bbox.right, bbox.top]]
raster_boundary = shape({'coordinates': [extent], 'type': 'Polygon'})
if not geometry.intersects(raster_boundary):
return
# get pixel coordinates of the geometry's bounding box,
ll = raster.index(
*geometry.bounds[0:2]) # lowerleft bounds[0:2] xmin, ymin
ur = raster.index(
*geometry.bounds[2:4]) # upperright bounds[2:4] xmax, ymax
# # create an affine transform for the subset data
# t = raster.transform
# shifted_affine = Affine(t.a, t.b, t.c + ll[1] * t.a, t.d, t.e, t.f + ur[0] * t.e)
#
# # read the subset of the data into a numpy array
# window = ((ur[0], ll[0] + 1), (ll[1], ur[1] + 1))
# when the shapefile polygon is larger than the raster
row_begin = ur[0] if ur[0] > 0 else 0
row_end = ll[0] + 1 if ll[0] > -1 else 0
col_begin = ll[1] if ll[1] > 0 else 0
col_end = ur[1] + 1 if ur[1] > -1 else 0
window = ((row_begin, row_end), (col_begin, col_end))
# create an affine transform for the subset data
t = raster.transform
shifted_affine = Affine(t.a, t.b, t.c + col_begin * t.a, t.d, t.e,
t.f + row_begin * t.e)
out_data = raster.read(window=window)
# check whether the numpy array is empty or not?
if out_data.size == 0 or np.all(out_data == raster.nodata):
print('the grid does not intersect with the raster')
return
# if len(out_data)==0:
# print('the grid does not intersect with the raster')
# return
with rasterio.open("/tmp/mask_%s.tif" % name,
'w',
driver='GTiff',
width=out_data.shape[2],
height=out_data.shape[1],
crs=raster.crs,
transform=shifted_affine,
dtype=np.uint16,
nodata=256,
count=raster.count,
indexes=raster.indexes) as dst:
# Write the src array into indexed bands of the dataset. If `indexes` is a list, the src must be a 3D array of matching shape. If an int, the src must be a 2D array.
dst.write(out_data.astype(rasterio.uint16), indexes=raster.indexes)
# bands_num = raster.count
# multi_bands_data = []
# for i in range(bands_num):
# # band_name = raster.indexes[i]
# data = raster.read(i + 1, window=window)
# # rasterize the geometry
# mask = rasterio.features.rasterize([(geometry, 0)], out_shape=data.shape, transform=shifted_affine, fill=1,
# all_touched=True, dtype=np.uint8)
#
# # create a masked numpy array
# masked_data = np.ma.array(data=data, mask=mask.astype(bool), dtype=np.float32)
# masked_data.data[masked_data.mask] = np.nan # raster.profile nodata is 256
# out_image = masked_data.data
# multi_bands_data.append(out_image)
# out_data = np.array(multi_bands_data)
# with rasterio.open("/tmp/mask_%s.tif" % name, 'w', driver='GTiff', width=out_data.shape[2],
# height=out_data.shape[1], crs=raster.crs, transform=shifted_affine, dtype=np.uint16, nodata=256,
# count=bands_num, indexes=raster.indexes) as dst:
# dst.write(out_image.astype(rasterio.uint16), 1)
# create a masked label numpy array
out_shape = ((window[0][1] - window[0][0]), (window[1][1] - window[1][0]))
mask = rasterio.features.rasterize([(geometry, 0)],
out_shape=out_shape,
transform=shifted_affine,
fill=1,
all_touched=True,
dtype=np.uint8)
label_array = np.empty(out_shape)
label_array[mask == 0] = tag
with rasterio.open("/tmp/label_%s.tif" % name,
'w',
driver='GTiff',
width=out_shape[1],
height=out_shape[0],
crs=raster.crs,
transform=shifted_affine,
dtype=np.uint16,
nodata=256,
count=1) as dst:
dst.write(label_array.astype(rasterio.uint16), 1)
def _xarray_affine(obj):
dims = obj.crs.dimensions
xres, xoff = data_resolution_and_offset(obj[dims[1]].values)
yres, yoff = data_resolution_and_offset(obj[dims[0]].values)
return Affine.translation(xoff, yoff) * Affine.scale(xres, yres)
metadata = landsat_metadata(name_dict['mtl'])
with rasterio.open(name_dict['red']) as raster:
affine_transform = raster.transform
ubc_col, ubc_row = ~affine_transform * (ubc_x, ubc_y)
ubc_col, ubc_row = int(ubc_col), int(ubc_row)
l_col_offset = -100
t_row_offset = -100
width = 200
height = 200
ul_col = ubc_col + l_col_offset
ul_row = ubc_row + t_row_offset
ul_x, ul_y = affine_transform * (ul_col, ul_row)
lr_x, lr_y = affine_transform * (ul_col + width, ul_row + height)
file_dict[season]['small_window'] = Window(ul_col, ul_row, width, height)
print(file_dict[season]['small_window'])
file_dict[season]['new_affine'] = Affine(30., 0., ul_x, 0., -30., ul_y)
file_dict[season]['extent'] = [ul_x, lr_x, lr_y, ul_y]
### END SOLUTION
# # The cell below calculates the reflectivities for red and nearir
# In[9]:
for season in ['spring', 'fall']:
name_dict = file_dict[season]['filenames']
metadata_string = str(name_dict['mtl'])
refl_dict = dict()
small_window = file_dict[season]['small_window']
for key in ['red', 'nearir']:
filepath = name_dict[key]
bandnum = landsat_band_dict[key]
