def test_fillnodata_mask_ones():
# when mask is all ones, image should be unmodified
a = np.ones([3, 3]) * 42
a[1][1] = 0
mask = np.ones([3, 3])
result = fillnodata(a, mask)
assert (np.all(a == result))
def interpolateRaster(dirpath, out_fp):
search_criteria = "*.tif"
q = os.path.join(dirpath, search_criteria)
# glob function can be used to list files from a directory with specific criteria
dem_fps = glob.glob(q)
print(len(dem_fps))
for fp in tqdm(dem_fps):
# print(fp)
# print("interplate_dtm_%s"%os.path.basename(fp))
with rasterio.open(fp, 'r') as src:
data = src.read(1, masked=True)
msk = src.read_masks(1)
#配置max_search_distance参数，或者多次执行插值，补全较大数据缺失区域
fill_raster = fillnodata(
data, msk, max_search_distance=400.0, smoothing_iterations=0
) #reference:https://rasterio.readthedocs.io/en/latest/api/rasterio.fill.html
# scaler=MinMaxScaler()
# plt.imshow(scaler.fit_transform(fill_raster))
# plt.imshow(scaler.fit_transform(data))
out_meta = src.meta.copy()
arr = np.random.randint(5, size=(100, 100)).astype(np.float)
with rasterio.open(
os.path.join(out_fp,
"interplate_dtm_%s" % os.path.basename(fp)),
"w", **out_meta) as dest:
dest.write(fill_raster, 1)
def fill_nodata(file_to_fill, mask_file=None, plot=False):
raster_filepath = os.path.dirname(file_to_fill) + "/"
raster_filename = os.path.basename(file_to_fill)
if mask_file:
with rasterio.open(mask_file) as src:
masks = src.read_masks()
count = src.count
mask = masks[0] & masks[1]
else:
mask = None
if plot:
plt.imshow(mask, cmap='gray')
plt.show()
with rasterio.open(file_to_fill) as src:
nodata = src.nodata
kwargs = src.meta
kwargs.update(dtype=rasterio.float32, count=1)
ndwi = src.read(1)
if plot:
plt.imshow(ndwi, cmap='gray')
plt.show()
filled = fillnodata(ndwi, (ndwi != nodata), max_search_distance=300)
if plot:
plt.imshow(filled, cmap='gray')
plt.show()
out_filename = raster_filepath + raster_filename.split(
sep=".")[0] + "_filled.tif"
with rasterio.open(out_filename, 'w', **kwargs) as dst:
dst.nodata = 0
dst.write_band(1, filled.astype(rasterio.float32))
def _fill_nodata(self):
"""Fill nodata values in elevation grid by interpolation.
Wrapper around GDAL/rasterio's FillNoData, fillnodata methods
"""
if ~np.isnan(self.nodata_value):
nodata_mask = self._griddata == self.nodata_value
else:
nodata_mask = np.isnan(self._griddata)
self.nodata_mask = nodata_mask
# XXX: GDAL (or rasterio) FillNoData takes mask with 0s at nodata
num_nodata = np.sum(nodata_mask)
prev_nodata = np.nan
while num_nodata > 0 or num_nodata == prev_nodata:
mask = np.isnan(self._griddata)
col_nodata = np.sum(mask, axis=0).max()
row_nodata = np.sum(mask, axis=1).max()
dist = max(row_nodata, col_nodata) / 2
self._griddata = fillnodata(self._griddata,
mask=~mask,
max_search_distance=dist)
prev_nodata = copy(num_nodata)
num_nodata = np.sum(np.isnan(self._griddata))
self.is_interpolated = True
def _fill_tile(r, c, row_min, row_max):
# rows = np.linspace(0, ids.shape[0], num_rows, dtype=int)
rows = np.linspace(row_min, row_max, num_rows, dtype=int)
cols = np.linspace(0, ids.shape[1], num_cols, dtype=int)
r_buffer_b = kernel_size if r != 0 else 0
r_buffer_t = kernel_size if r != num_rows - 2 else 0
c_buffer_l = kernel_size if c != 0 else 0
c_buffer_r = kernel_size if c != num_cols - 2 else 0
window_write = ((rows[r], rows[r + 1]), (cols[c], cols[c + 1]))
window_read = ((rows[r] - r_buffer_b, rows[r + 1] + r_buffer_t),
(cols[c] - c_buffer_l, cols[c + 1] + c_buffer_r))
print('Write window {}'.format(window_write))
print('Read window due to patch {}'.format(window_read))
tile_data = ids.read(1, masked=True, window=window_read)
print(tile_data.shape)
if tile_data.count(
) == tile_data.size: # all unmasked pixels, nothing to do
orig_data = ids.read(1, masked=True, window=window_write)
return Tile(data=orig_data, window=window_write)
elif tile_data.count() == 0: # all masked pixels, can't do filling
orig_data = ids.read(1, masked=True, window=window_write)
return Tile(data=orig_data, window=window_write)
data_filled = fillnodata(tile_data,
mask=None,
max_search_distance=kernel_size)
# return r_buffer_b, r_buffer_t, c_buffer_l, c_buffer_r, data_filled, window_write
data_write = data_filled[r_buffer_b:data_filled.shape[0] - r_buffer_t,
c_buffer_l:data_filled.shape[1] - c_buffer_r]
return Tile(data=data_write, window=window_write)
def test_fillnodata_mask_ones():
# when mask is all ones, image should be unmodified
a = numpy.ones([3, 3]) * 42
a[1][1] = 0
mask = numpy.ones([3, 3])
result = fillnodata(a, mask)
assert(numpy.all(a == result))
def findPatch(heightmap, dbg=False):
mask = heightmap > 0
filled = binary_fill_holes(mask)
filled_pixels = filled.astype(np.float32) - mask.astype(np.float32)
if dbg:
img(filled_pixels, os.path.join(OUTPUT, '_pixels_to_interpolate.png'))
img(heightmap, os.path.join(OUTPUT, '_heightmap.png'))
interpolated_holes = fillnodata(heightmap, mask=filled_pixels == 0)
if dbg:
img(interpolated_holes, os.path.join(OUTPUT,
'_interpolated_holes.png'))
edt = distance_transform_edt(filled)
if dbg:
img(edt, os.path.join(OUTPUT, '_edt.png'))
patch_size = edt.max() * np.sqrt(2)
coord = np.unravel_index(edt.argmax(), edt.shape)
return patch_size, coord
def interpolate(raster, max_search_distance=10):
with rasterio.open(raster) as src:
profile = src.profile
arr = src.read(1)
arr_filled = fillnodata(arr,
mask=src.read_masks(1),
max_search_distance=max_search_distance)
with rasterio.open(raster, 'w', **profile) as dest:
dest.write_band(1, arr_filled)
def test_fillnodata():
"""Test filling nodata values in an ndarray"""
# create a 5x5 array, with some missing data
a = numpy.ones([3, 3]) * 42
a[1][1] = 0
# find the missing data
mask = ~(a == 0)
# fill the missing data using interpolation from the edges
result = fillnodata(a, mask)
assert(numpy.all((numpy.ones([3, 3]) * 42) == result))
def test_fillnodata():
"""Test filling nodata values in an ndarray"""
# create a 5x5 array, with some missing data
a = np.ones([3, 3]) * 42
a[1][1] = 0
# find the missing data
mask = ~(a == 0)
# fill the missing data using interpolation from the edges
result = fillnodata(a, mask)
assert (np.all((np.ones([3, 3]) * 42) == result))
def fill_nodata(self):
"""
A parallelized version is presented here:
https://github.com/basaks/rasterio/blob/master/examples/fill_large_raster.py
"""
tmpfile = tempfile.NamedTemporaryFile(prefix=tmpdir)
with rasterio.open(tmpfile.name, 'w', **self._src.meta.copy()) as dst:
for window in self.iter_windows():
dst.write(fillnodata(self._src.read(window=window,
masked=True)),
window=window)
self._tmpfile = tmpfile
def thicket_agc_post_proc(image_mapper=ImageMapper()):
"""
Post process thicket AGC map - helper function for MsImageMapper
Writes out a cleaned version of map generated by ImageMapper.map(...), ills nodata and places sensible limits on
AGC values.
Parameters
----------
image_mapper : ImageMapper
instance of ImageMapper that has generated map
Returns
-------
post processed raster file name
"""
with rasterio.Env():
with rasterio.open(image_mapper.map_file_name, 'r') as in_ds:
out_profile = in_ds.profile
out_profile.update(count=1)
split_ext = os.path.splitext(image_mapper.map_file_name)
out_file_name = '{0}_postproc{1}'.format(split_ext[0],
split_ext[1])
with rasterio.open(out_file_name, 'w', **out_profile) as out_ds:
if (not out_profile['tiled']) or (np.prod(in_ds.shape) < 10e6):
in_windows = enumerate(
[Window(0, 0, in_ds.width,
in_ds.height)]) # read whole raster at once
else:
in_windows = in_ds.block_windows(1) # read in blocks
for ji, block_win in in_windows:
in_block = in_ds.read(1, window=block_win, masked=True)
in_block[in_block < 0] = 0
in_block.mask = (in_block.mask.astype(np.bool) |
(in_block > 95) | (in_block < 0)).astype(
rasterio.uint8)
in_mask = in_block.mask.copy()
sieved_msk = sieve(in_mask.astype(rasterio.uint8),
size=2000)
out_block = fill.fillnodata(in_block,
mask=None,
max_search_distance=20,
smoothing_iterations=1)
out_block[sieved_msk.astype(np.bool)] = image_mapper.nodata
out_ds.write(out_block, indexes=1, window=block_win)
return out_file_name
def mask_raster(raster_path, mask_path, crs):
with fiona.open(mask_path, "r") as shapefile:
shapes = [feature["geometry"] for feature in shapefile]
with rasterio.open(raster_path) as src:
out_image, out_transform = rasterio.mask.mask(src, shapes, crop=True)
out_image[out_image < 0] = np.nan
mask = (out_image != 0)
out_image = fillnodata(out_image, mask)
out_meta = src.meta
out_meta.update({
"driver": "GTiff",
"height": out_image.shape[1],
"width": out_image.shape[2],
"transform": out_transform,
"crs": crs
})
return out_image, out_meta
def fill_internal_nodata(img: str, out: str, aoi: str) -> None:
"""
Fills internal NoData gaps by interpolating across them.
'Interal' gaps are definied as those within aoi polygon.
TODO: Areas outside of AOI polygon are masked regardless of
if they are NoData are not, so this effectively only works
if the img is clipped to the AOI already. Workaround would
be to create the aoi on the fly from the edges of the valid
data in the img.
"""
temp_filled = r'/vsimem/temp_filled.tif'
with rio.open(img) as src:
with rio.Env():
profile = src.profile
with rio.open(temp_filled, 'w', **profile) as dst:
for i in range(src.count):
b = i + 1
arr = src.read(b)
mask = src.read_masks(b)
filled = fillnodata(arr, mask=mask)
dst.write(filled.astype(src.dtypes[i]), b)
# with fiona.open(aoi) as shapefile:
# shapes = [feature['geometry'] for feature in shapefile]
gdf = read_vec(aoi)
shapes = gdf.geometry.values
# Warn if not same CRS
with rio.open(temp_filled) as src:
if gdf.crs.to_wkt() != src.crs.to_wkt():
logger.warning('AOI and raster to-be-filled do not have matching'
'CRS:\nAOI:{}\nRaster:{}'.format(gdf.crs, src.crs))
out_img, out_trans = rasterio.mask.mask(src, shapes)
with rio.open(out, 'w', **profile) as dst:
dst.write(out_img)
dest_dir = '/media/rmsare/GALLIUMOS/data/ot_data/nsaf_6km/'
files = os.listdir(proc_dir)
files = sort_by_utm_northing(files)
pbar = ProgressBar(widgets=[Percentage(), ' ',
Bar(), ' ',
ETA()],
maxval=len(files))
pbar.start()
for i, f in enumerate(files):
padded_data = pad_with_neighboring_values(f, pad, data_dir=source_dir)
nodata_mask = ~np.isnan(padded_data)
num_nodata = np.sum(~nodata_mask)
while num_nodata > 0:
padded_data = fillnodata(padded_data, mask=nodata_mask)
nodata_mask = ~np.isnan(padded_data)
num_nodata = np.sum(~nodata_mask)
nrows, ncols = padded_data.shape
data_file = source_dir + f
inraster = gdal.Open(data_file)
transform = inraster.GetGeoTransform()
driver = gdal.GetDriverByName('GTiff')
outraster = driver.Create(dest_dir + f, ncols, nrows, 1,
gdal.GDT_Float32)
outraster.SetGeoTransform(transform)
out_band = outraster.GetRasterBand(1)
out_band.WriteArray(padded_data)
def calc(ctx, command, files, output, name, dtype, masked, overwrite, mem_limit, creation_options):
"""A raster data calculator
Evaluates an expression using input datasets and writes the result
to a new dataset.
Command syntax is lisp-like. An expression consists of an operator
or function name and one or more strings, numbers, or expressions
enclosed in parentheses. Functions include ``read`` (gets a raster
array) and ``asarray`` (makes a 3-D array from 2-D arrays).
\b
* (read i) evaluates to the i-th input dataset (a 3-D array).
* (read i j) evaluates to the j-th band of the i-th dataset (a
2-D array).
* (take foo j) evaluates to the j-th band of a dataset named foo
(see help on the --name option above).
* Standard numpy array operators (+, -, *, /) are available.
* When the final result is a list of arrays, a multiple band
output file is written.
* When the final result is a single array, a single band output
file is written.
Example:
\b
$ rio calc "(+ 2 (* 0.95 (read 1)))" tests/data/RGB.byte.tif \\
> /tmp/out.tif
The command above produces a 3-band GeoTIFF with all values scaled
by 0.95 and incremented by 2.
\b
$ rio calc "(asarray (+ 125 (read 1)) (read 1) (read 1))" \\
> tests/data/shade.tif /tmp/out.tif
The command above produces a 3-band RGB GeoTIFF, with red levels
incremented by 125, from the single-band input.
The maximum amount of memory used to perform caculations defaults to
64 MB. This number can be increased to improve speed of calculation.
"""
import numpy as np
try:
with ctx.obj['env']:
output, files = resolve_inout(files=files, output=output,
overwrite=overwrite)
inputs = ([tuple(n.split('=')) for n in name] +
[(None, n) for n in files])
sources = [rasterio.open(path) for name, path in inputs]
first = sources[0]
kwargs = first.profile
kwargs.update(**creation_options)
dtype = dtype or first.meta['dtype']
kwargs['dtype'] = dtype
# Extend snuggs.
snuggs.func_map['read'] = _read_array
snuggs.func_map['band'] = lambda d, i: _get_bands(inputs, sources, d, i)
snuggs.func_map['bands'] = lambda d: _get_bands(inputs, sources, d)
snuggs.func_map['fillnodata'] = lambda *args: fillnodata(*args)
snuggs.func_map['sieve'] = lambda *args: sieve(*args)
dst = None
# The windows iterator is initialized with a single sample.
# The actual work windows will be added in the second
# iteration of the loop.
work_windows = [(None, Window(0, 0, 16, 16))]
for ij, window in work_windows:
ctxkwds = OrderedDict()
for i, ((name, path), src) in enumerate(zip(inputs, sources)):
# Using the class method instead of instance
# method. Latter raises
#
# TypeError: astype() got an unexpected keyword
# argument 'copy'
#
# possibly something to do with the instance being
# a masked array.
ctxkwds[name or '_i%d' % (i + 1)] = src.read(masked=masked, window=window)
res = snuggs.eval(command, **ctxkwds)
if (isinstance(res, np.ma.core.MaskedArray) and (
tuple(LooseVersion(np.__version__).version) < (1, 9) or
tuple(LooseVersion(np.__version__).version) > (1, 10))):
res = res.filled(kwargs['nodata'])
if len(res.shape) == 3:
results = np.ndarray.astype(res, dtype, copy=False)
else:
results = np.asanyarray(
[np.ndarray.astype(res, dtype, copy=False)])
# The first iteration is only to get sample results and from them
# compute some properties of the output dataset.
if dst is None:
kwargs['count'] = results.shape[0]
dst = rasterio.open(output, 'w', **kwargs)
work_windows.extend(_chunk_output(dst.width, dst.height, dst.count, np.dtype(dst.dtypes[0]).itemsize, mem_limit=mem_limit))
# In subsequent iterations we write results.
else:
dst.write(results, window=window)
except snuggs.ExpressionError as err:
click.echo("Expression Error:")
click.echo(' %s' % err.text)
click.echo(' ' + ' ' * err.offset + "^")
click.echo(err)
raise click.Abort()
finally:
if dst:
dst.close()
for src in sources:
src.close()
def test_fillnodata_invalid_types():
a = numpy.ones([3, 3])
with pytest.raises(ValueError):
fillnodata(None, a)
with pytest.raises(ValueError):
fillnodata(a, 42)
def calc(ctx, command, files, name, dtype):
"""A raster data calculator
Evaluates an expression using input datasets and writes the result
to a new dataset.
Command syntax is lisp-like. An expression consists of an operator
or function name and one or more strings, numbers, or expressions
enclosed in parentheses. Functions include ``read`` (gets a raster
array) and ``asarray`` (makes a 3-D array from 2-D arrays).
\b
* (read i) evaluates to the i-th input dataset (a 3-D array).
* (read i j) evaluates to the j-th band of the i-th dataset (a 2-D
array).
* (take foo j) evaluates to the j-th band of a dataset named foo (see
help on the --name option above).
* Standard numpy array operators (+, -, *, /) are available.
* When the final result is a list of arrays, a multi band output
file is written.
* When the final result is a single array, a single band output
file is written.
Example:
\b
$ rio calc "(+ 2 (* 0.95 (read 1)))" tests/data/RGB.byte.tif \\
> /tmp/out.tif
Produces a 3-band GeoTIFF with all values scaled by 0.95 and
incremented by 2.
\b
$ rio calc "(asarray (+ 125 (read 1)) (read 1) (read 1))" \\
> tests/data/shade.tif /tmp/out.tif
Produces a 3-band RGB GeoTIFF, with red levels incremented by 125,
from the single-band input.
"""
import numpy as np
verbosity = (ctx.obj and ctx.obj.get('verbosity')) or 1
logger = logging.getLogger('rio')
try:
with rasterio.drivers(CPL_DEBUG=verbosity > 2):
output = files[-1]
inputs = ([tuple(n.split('=')) for n in name] +
[(None, n) for n in files[:-1]])
with rasterio.open(inputs[0][1]) as first:
kwargs = first.meta
kwargs['transform'] = kwargs.pop('affine')
dtype = dtype or first.meta['dtype']
kwargs['dtype'] = dtype
ctxkwds = {}
for i, (name, path) in enumerate(inputs):
with rasterio.open(path) as src:
# Using the class method instead of instance
# method. Latter raises
#
# TypeError: astype() got an unexpected keyword
# argument 'copy'
#
# possibly something to do with the instance being
# a masked array.
ctxkwds[name or '_i%d' % (i+1)] = src.read()
# Extend snuggs.
snuggs.func_map['read'] = read_array
snuggs.func_map['band'] = lambda d, i: get_bands(inputs, d, i)
snuggs.func_map['bands'] = lambda d: get_bands(inputs, d)
snuggs.func_map['fillnodata'] = lambda *args: fillnodata(*args)
snuggs.func_map['sieve'] = lambda *args: sieve(*args)
res = snuggs.eval(command, **ctxkwds)
if len(res.shape) == 3:
results = np.ndarray.astype(res, dtype, copy=False)
else:
results = np.asanyarray(
[np.ndarray.astype(res, dtype, copy=False)])
kwargs['count'] = results.shape[0]
with rasterio.open(output, 'w', **kwargs) as dst:
dst.write(results)
sys.exit(0)
except snuggs.ExpressionError as err:
click.echo("Expression Error:")
click.echo(' %s' % err.text)
click.echo(' ' + ' ' * err.offset + "^")
click.echo(err)
sys.exit(1)
except Exception as err:
t, v, tb = sys.exc_info()
for line in traceback.format_exception_only(t, v):
click.echo(line, nl=False)
sys.exit(1)
def calc(ctx, command, files, output, name, dtype, masked, overwrite,
creation_options):
"""A raster data calculator
Evaluates an expression using input datasets and writes the result
to a new dataset.
Command syntax is lisp-like. An expression consists of an operator
or function name and one or more strings, numbers, or expressions
enclosed in parentheses. Functions include ``read`` (gets a raster
array) and ``asarray`` (makes a 3-D array from 2-D arrays).
\b
* (read i) evaluates to the i-th input dataset (a 3-D array).
* (read i j) evaluates to the j-th band of the i-th dataset (a 2-D
array).
* (take foo j) evaluates to the j-th band of a dataset named foo (see
help on the --name option above).
* Standard numpy array operators (+, -, *, /) are available.
* When the final result is a list of arrays, a multi band output
file is written.
* When the final result is a single array, a single band output
file is written.
Example:
\b
$ rio calc "(+ 2 (* 0.95 (read 1)))" tests/data/RGB.byte.tif \\
> /tmp/out.tif
Produces a 3-band GeoTIFF with all values scaled by 0.95 and
incremented by 2.
\b
$ rio calc "(asarray (+ 125 (read 1)) (read 1) (read 1))" \\
> tests/data/shade.tif /tmp/out.tif
Produces a 3-band RGB GeoTIFF, with red levels incremented by 125,
from the single-band input.
"""
import numpy as np
try:
with ctx.obj['env']:
output, files = resolve_inout(files=files, output=output,
overwrite=overwrite)
inputs = ([tuple(n.split('=')) for n in name] +
[(None, n) for n in files])
with rasterio.open(inputs[0][1]) as first:
kwargs = first.meta
kwargs.update(**creation_options)
dtype = dtype or first.meta['dtype']
kwargs['dtype'] = dtype
ctxkwds = OrderedDict()
for i, (name, path) in enumerate(inputs):
with rasterio.open(path) as src:
# Using the class method instead of instance
# method. Latter raises
#
# TypeError: astype() got an unexpected keyword
# argument 'copy'
#
# possibly something to do with the instance being
# a masked array.
ctxkwds[name or '_i%d' % (i + 1)] = src.read(masked=masked)
# Extend snuggs.
snuggs.func_map['read'] = read_array
snuggs.func_map['band'] = lambda d, i: get_bands(inputs, d, i)
snuggs.func_map['bands'] = lambda d: get_bands(inputs, d)
snuggs.func_map['fillnodata'] = lambda *args: fillnodata(*args)
snuggs.func_map['sieve'] = lambda *args: sieve(*args)
res = snuggs.eval(command, ctxkwds)
if (isinstance(res, np.ma.core.MaskedArray) and (
tuple(LooseVersion(np.__version__).version) < (1, 9) or
tuple(LooseVersion(np.__version__).version) > (1, 10))):
res = res.filled(kwargs['nodata'])
if len(res.shape) == 3:
results = np.ndarray.astype(res, dtype, copy=False)
else:
results = np.asanyarray(
[np.ndarray.astype(res, dtype, copy=False)])
kwargs['count'] = results.shape[0]
with rasterio.open(output, 'w', **kwargs) as dst:
dst.write(results)
except snuggs.ExpressionError as err:
click.echo("Expression Error:")
click.echo(' %s' % err.text)
click.echo(' ' + ' ' * err.offset + "^")
click.echo(err)
raise click.Abort()
#plt.imshow(img)
ent = entropy(ndem, disk(2))
maske=ent<1.5
if plotear:
plt.imshow(maske)
#%% Etapa 1. Cierre: generación de la máscara como intersección de ambas máscaras
mask=maske*mask3
if plotear:
plt.imshow(mask)
#%% Etapa 2. Interpolación
# borro los datos fuera de la máscara
# y completo los faltantes con un peso inverso a la distancia y radio de 100
mdt=fillnodata(dem*mask,mask*1, max_search_distance=100)
if plotear:
plt.imshow(mdt)
#%%
#diferentes cosas para ver
if plotear:
plt.imshow(mdt)
plt.imshow(mask3)
plt.imshow(maske)
plt.imshow(mask)
plt.figure()
plt.imshow(mdt,vmin=hmin,vmax=30)
def get_aot(self,
data,
sza,
meta,
data_values='dn',
angle_factor=0.01,
dn_interp=None,
interp_method='fast',
aot_fallback=0.3,
h2o=2.0,
o3=0.3,
altitude=0.0,
w=None,
n_jobs=1):
"""
Gets the aerosol optical thickness (AOT) from dark objects
Args:
data (DataArray): The digital numbers or top of atmosphere reflectance at a coarse resolution.
sza (float | DataArray): The solar zenith angle.
meta (Optional[namedtuple]): A metadata object with gain and bias coefficients.
data_values (Optional[str]): The values of ``data``. Choices are ['dn', 'toar'].
angle_factor (Optional[float]): The scale factor for angles.
dn_interp (Optional[DataArray]): A source ``DataArray`` at the target resolution.
interp_method (Optional[str]): The LUT interpolation method. Choices are ['fast', 'slow'].
'fast': Uses nearest neighbor lookup with ``scipy.interpolate.NearestNDInterpolator``.
'slow': Uses linear interpolation with ``scipy.interpolate.LinearNDInterpolator``.
aot_fallback (Optional[float | DataArray]): The aerosol optical thickness fallback if no dark objects
are found (unitless). [0,3].
h2o (Optional[float]): The water vapor (g/m^2). [0,8.5].
o3 (Optional[float]): The ozone (cm-atm). [0,8].
altitude (Optional[float]): The altitude over the sensor acquisition location (km above sea level).
w (Optional[int]): The smoothing window size (in pixels).
n_jobs (Optional[int]): The number of parallel jobs for ``moving_window`` and ``dask.compute``.
Returns:
``xarray.DataArray``:
Data range: 0-3
References:
See :cite:`masek_etal_2006`, :cite:`kaufman_etal_1997`, and :cite:`ouaidrari_vermote_1999`.
"""
if data_values not in ['dn', 'toar']:
logger.exception(" The data values should be 'dn' or 'toar'")
raise NameError
if isinstance(sza, xr.DataArray):
sza = sza.squeeze().data.compute(num_workers=n_jobs)
sza *= angle_factor
band_names = data.band.values.tolist()
doy = meta.date_acquired.timetuple().tm_yday
if data_values == 'dn':
m_p = coeffs_to_array(meta.m_p, band_names)
a_p = coeffs_to_array(meta.a_p, band_names)
m_l = coeffs_to_array(meta.m_l, band_names)
a_l = coeffs_to_array(meta.a_l, band_names)
toar = self.dn_to_toar(data, m_p, a_p, sun_angle=False)
rad = self.dn_to_radiance(data, m_l, a_l)
else:
toar = data
rad = self.toar_to_rad(data, meta)
# Get the SWIR2 band TOAR
swir2_toar = toar.sel(band='swir2')
# Get the blue band Radiance
blue_rad = rad.sel(band='blue')
# Get SWIR2 TOAR dark pixels
swir2_toar_dark = xr.where((swir2_toar >= 0.01) & (swir2_toar <= 0.15),
swir2_toar, np.nan)
blue_rad_dark = xr.where((swir2_toar >= 0.01) & (swir2_toar <= 0.15),
blue_rad, np.nan)
# Estimate the blue surface reflectance with
# a simple linear transformation (Masek et al., 2006)
blue_p = swir2_toar_dark * 0.33
# Get reflectance and radiance data as numpy arrays
blue_p_data = blue_p.squeeze().data.compute(num_workers=n_jobs)
blue_rad_dark_data = blue_rad_dark.squeeze().data.compute(
num_workers=n_jobs)
valid_idx = np.where(~np.isnan(blue_p_data))
if valid_idx[0].shape[0] > 0:
aot = self.get_optimized_aot(blue_rad_dark_data, blue_p_data,
meta.sensor, 'blue', interp_method,
sza, doy, h2o, o3, altitude)
mask = np.ones(aot.shape, dtype='uint8')
mask[np.isnan(blue_p_data)] = 0
aot = fillnodata(aot, mask=mask, max_search_distance=100)
if isinstance(dn_interp, xr.DataArray):
aot = self._resize(aot, dn_interp, w, n_jobs)
return ndarray_to_xarray(dn_interp, aot, ['aot'])
else:
return ndarray_to_xarray(data, aot, ['aot'])
else:
if isinstance(dn_interp, xr.DataArray):
return ndarray_to_xarray(
dn_interp,
np.zeros((dn_interp.gw.nrows, dn_interp.gw.ncols),
dtype='float64') + aot_fallback, ['aot'])
else:
return ndarray_to_xarray(
data,
np.zeros((data.gw.nrows, data.gw.ncols), dtype='float64') +
aot_fallback, ['aot'])
def fillna_in_raster(dir: str,
aqi_tif_name: str,
na_val: float = 1.0,
log: Logger = None) -> bool:
"""Fills nodata values in a raster by interpolating values from surrounding cells.
Value 1.0 is considered as nodata. If no nodata is found with that value, a small offset will be applied,
as sometimes the nodata value is slightly higher than 1.0 (assumably due to inaccuracy in netcdf to
geotiff conversion).
Args:
aqi_tif_name: The name of a raster file to be processed (in aqi_cache directory).
na_val: A value that represents nodata in the raster.
"""
# open AQI band from AQI raster file
aqi_filepath = dir + aqi_tif_name
aqi_raster = rasterio.open(aqi_filepath)
aqi_band = aqi_raster.read(1)
# create a nodata mask (map nodata values to 0)
# nodata value may be slightly higher than 1.0, hence try different offsets
na_offset = 0
for offset in [0.0, 0.01, 0.02, 0.04, 0.06, 0.08, 0.1, 0.12]:
na_offset = na_val + offset
nodata_count = np.sum(aqi_band <= na_offset)
if log:
log.info(f'Nodata offset: {offset} / nodata count: {nodata_count}')
# check if nodata values can be mapped with the current offset
if (nodata_count > 180000):
break
if (nodata_count < 180000):
if log:
log.info(
f'Failed to set nodata values in the AQI tif, nodata count: {nodata_count}'
)
aqi_nodata_mask = np.where(aqi_band <= na_offset, 0, aqi_band)
# fill nodata in aqi_band using nodata mask
aqi_band_fillna = fill.fillnodata(aqi_band, mask=aqi_nodata_mask)
# validate AQI values after na fill
invalid_count = np.sum(aqi_band_fillna < 1.0)
if (invalid_count > 0):
if log:
log.warning(
f'AQI band has {invalid_count} below 1 aqi values after na fill'
)
# write raster with filled nodata
aqi_raster_fillna = rasterio.open(aqi_filepath,
'w',
driver='GTiff',
height=aqi_raster.shape[0],
width=aqi_raster.shape[1],
count=1,
dtype='float32',
transform=aqi_raster.transform,
crs=aqi_raster.crs)
aqi_raster_fillna.write(aqi_band_fillna, 1)
aqi_raster_fillna.close()
return True
def mosaic_tiles(dir_tiles,
roi_xmin,
roi_ymin,
roi_xmax,
roi_ymax,
roi_transform,
roi_width,
roi_length,
roi_crs,
tile_dtype,
nodata_value,
kbl_dir,
kbl_scale=0,
query_string="",
resampling="nearest",
fill_nodata=True,
mosaic_filename=None):
"""
Combine KBL tiles into mosaic
Inputs
dir_tiles: str
directory to image tiles
roi_xmin, roi_ymin, roi_xmax, roi_ymax: float
coordinates of ROI
roi_transform: rasterio transform
ROI transform
roi_width: int
ROI width (no. of columns)
roi_length: int
ROI length (no. of rows)
roi_crs: str
CRS in which ROI limits are expressed (format "EPSG:code")
tile_dtype: numpy dtype
mosaic datatype
nodata_value: int/float
value for no data
kbl_scale: int
KBL scale (0, 8, 16)
query_string: str
query string to search for tile files
resampling: str
resampling mode for warp_to_transform
fill_nodata: bool
whether to fill no data areas
mosaic_filename: str
path to store mosaic
Outputs
mos: nd array
mosaic image
"""
tiles_id = find_tiles_kbl(roi_xmin,
roi_ymin,
roi_xmax,
roi_ymax,
roi_crs,
kbl_dir,
kbl_scale=kbl_scale)
if kbl_scale == 0: # TODO: add for kbl_scale 4
tiles = [
glob(
os.path.join(dir_tiles,
"{}*{:02d}.tif".format(query_string, int(i))))
for i in tiles_id
]
elif kbl_scale == 8:
tiles = [
glob(
os.path.join(
dir_tiles, "{}*{:03d}.tif".format(query_string,
int(i.replace("/",
"")))))
for i in tiles_id
]
elif kbl_scale == 16:
tiles = [
glob(
os.path.join(
dir_tiles,
"{}*{:03d}{}.tif".format(query_string,
int(i[:-1].replace("/", "")),
i[-1].lower()))) for i in tiles_id
]
tiles = [val for sublist in tiles for val in sublist]
if len(tiles) == 0:
print("Error! No tiles covering ROI found.")
return None
mos = np.ones((roi_length, roi_width), dtype=tile_dtype) * nodata_value
for tile in tiles:
tile_arr, tile_nd = warp_to_transform(os.path.join(dir_tiles, tile),
roi_transform,
roi_width,
roi_length,
roi_crs,
return_nodata=True,
resampling=resampling)
tile_arr[tile_arr == tile_nd] = nodata_value
mos[(mos == nodata_value)
& (tile_arr != tile_nd)] = tile_arr[(mos == nodata_value)
& (tile_arr != tile_nd)]
if fill_nodata & (np.sum(mos == nodata_value) > 0):
mos = fill.fillnodata(mos,
mos != nodata_value,
max_search_distance=100.0,
smoothing_iterations=0)
if mosaic_filename is not None:
ar2tif(mos,
mosaic_filename,
roi_crs,
roi_transform,
dtype=mos.dtype,
band_names=["LC"],
nodata_value=nodata_value)
return mos
def test_fillnodata(hole_in_ones):
"""Test filling nodata values in an ndarray"""
mask = hole_in_ones == 1
result = fillnodata(hole_in_ones, mask)
assert (result == 1).all()
def calc(ctx, command, files, output, name, dtype, masked, creation_options):
"""A raster data calculator
Evaluates an expression using input datasets and writes the result
to a new dataset.
Command syntax is lisp-like. An expression consists of an operator
or function name and one or more strings, numbers, or expressions
enclosed in parentheses. Functions include ``read`` (gets a raster
array) and ``asarray`` (makes a 3-D array from 2-D arrays).
\b
* (read i) evaluates to the i-th input dataset (a 3-D array).
* (read i j) evaluates to the j-th band of the i-th dataset (a 2-D
array).
* (take foo j) evaluates to the j-th band of a dataset named foo (see
help on the --name option above).
* Standard numpy array operators (+, -, *, /) are available.
* When the final result is a list of arrays, a multi band output
file is written.
* When the final result is a single array, a single band output
file is written.
Example:
\b
$ rio calc "(+ 2 (* 0.95 (read 1)))" tests/data/RGB.byte.tif \\
> /tmp/out.tif
Produces a 3-band GeoTIFF with all values scaled by 0.95 and
incremented by 2.
\b
$ rio calc "(asarray (+ 125 (read 1)) (read 1) (read 1))" \\
> tests/data/shade.tif /tmp/out.tif
Produces a 3-band RGB GeoTIFF, with red levels incremented by 125,
from the single-band input.
"""
import numpy as np
verbosity = (ctx.obj and ctx.obj.get('verbosity')) or 1
logger = logging.getLogger('rio')
try:
with rasterio.drivers(CPL_DEBUG=verbosity > 2):
output, files = resolve_inout(files=files, output=output)
inputs = ([tuple(n.split('=')) for n in name] + [(None, n)
for n in files])
with rasterio.open(inputs[0][1]) as first:
kwargs = first.meta
kwargs.update(**creation_options)
kwargs['transform'] = kwargs.pop('affine')
dtype = dtype or first.meta['dtype']
kwargs['dtype'] = dtype
ctxkwds = {}
for i, (name, path) in enumerate(inputs):
with rasterio.open(path) as src:
# Using the class method instead of instance
# method. Latter raises
#
# TypeError: astype() got an unexpected keyword
# argument 'copy'
#
# possibly something to do with the instance being
# a masked array.
ctxkwds[name or '_i%d' % (i + 1)] = src.read(masked=masked)
# Extend snuggs.
snuggs.func_map['read'] = read_array
snuggs.func_map['band'] = lambda d, i: get_bands(inputs, d, i)
snuggs.func_map['bands'] = lambda d: get_bands(inputs, d)
snuggs.func_map['fillnodata'] = lambda *args: fillnodata(*args)
snuggs.func_map['sieve'] = lambda *args: sieve(*args)
res = snuggs.eval(command, **ctxkwds)
if (isinstance(res, np.ma.core.MaskedArray)
and tuple(LooseVersion(np.__version__).version) <
(1, 9, 0)):
res = res.filled(kwargs['nodata'])
if len(res.shape) == 3:
results = np.ndarray.astype(res, dtype, copy=False)
else:
results = np.asanyarray(
[np.ndarray.astype(res, dtype, copy=False)])
kwargs['count'] = results.shape[0]
with rasterio.open(output, 'w', **kwargs) as dst:
dst.write(results)
except snuggs.ExpressionError as err:
click.echo("Expression Error:")
click.echo(' %s' % err.text)
click.echo(' ' + ' ' * err.offset + "^")
click.echo(err)
raise click.Abort()
def test_fillnodata_masked_array(hole_in_ones):
"""Test filling nodata values in a masked ndarray"""
ma = np.ma.masked_array(hole_in_ones, (hole_in_ones == 0))
result = fillnodata(ma)
assert (result == 1).all()
def fill_nodata(img, mask, fillBands, maxSearchDistance):
for b in fillBands:
img[b - 1] = fillnodata(img[b - 1], mask, maxSearchDistance)
return img
def test_fillnodata_invalid_types():
a = np.ones([3, 3])
with pytest.raises(ValueError):
fillnodata(None, a)
with pytest.raises(ValueError):
fillnodata(a, 42)
def calc(ctx, command, files, output, name, dtype, masked, overwrite,
mem_limit, creation_options):
"""A raster data calculator
Evaluates an expression using input datasets and writes the result
to a new dataset.
Command syntax is lisp-like. An expression consists of an operator
or function name and one or more strings, numbers, or expressions
enclosed in parentheses. Functions include ``read`` (gets a raster
array) and ``asarray`` (makes a 3-D array from 2-D arrays).
\b
* (read i) evaluates to the i-th input dataset (a 3-D array).
* (read i j) evaluates to the j-th band of the i-th dataset (a
2-D array).
* (take foo j) evaluates to the j-th band of a dataset named foo
(see help on the --name option above).
* Standard numpy array operators (+, -, *, /) are available.
* When the final result is a list of arrays, a multiple band
output file is written.
* When the final result is a single array, a single band output
file is written.
Example:
\b
$ rio calc "(+ 2 (* 0.95 (read 1)))" tests/data/RGB.byte.tif \\
> /tmp/out.tif
The command above produces a 3-band GeoTIFF with all values scaled
by 0.95 and incremented by 2.
\b
$ rio calc "(asarray (+ 125 (read 1)) (read 1) (read 1))" \\
> tests/data/shade.tif /tmp/out.tif
The command above produces a 3-band RGB GeoTIFF, with red levels
incremented by 125, from the single-band input.
The maximum amount of memory used to perform caculations defaults to
64 MB. This number can be increased to improve speed of calculation.
"""
import numpy as np
dst = None
sources = []
try:
with ctx.obj['env']:
output, files = resolve_inout(files=files,
output=output,
overwrite=overwrite)
inputs = ([tuple(n.split('=')) for n in name] + [(None, n)
for n in files])
sources = [rasterio.open(path) for name, path in inputs]
first = sources[0]
kwargs = first.profile
kwargs.update(**creation_options)
dtype = dtype or first.meta['dtype']
kwargs['dtype'] = dtype
# Extend snuggs.
snuggs.func_map['read'] = _read_array
snuggs.func_map['band'] = lambda d, i: _get_bands(
inputs, sources, d, i)
snuggs.func_map['bands'] = lambda d: _get_bands(inputs, sources, d)
snuggs.func_map['fillnodata'] = lambda *args: fillnodata(*args)
snuggs.func_map['sieve'] = lambda *args: sieve(*args)
# The windows iterator is initialized with a single sample.
# The actual work windows will be added in the second
# iteration of the loop.
work_windows = [(None, Window(0, 0, 16, 16))]
for ij, window in work_windows:
ctxkwds = OrderedDict()
for i, ((name, path), src) in enumerate(zip(inputs, sources)):
# Using the class method instead of instance
# method. Latter raises
#
# TypeError: astype() got an unexpected keyword
# argument 'copy'
#
# possibly something to do with the instance being
# a masked array.
ctxkwds[name or '_i%d' % (i + 1)] = src.read(masked=masked,
window=window)
res = snuggs.eval(command, **ctxkwds)
if (isinstance(res, np.ma.core.MaskedArray) and
(tuple(LooseVersion(np.__version__).version) <
(1, 9) or tuple(LooseVersion(np.__version__).version) >
(1, 10))):
res = res.filled(kwargs['nodata'])
if len(res.shape) == 3:
results = np.ndarray.astype(res, dtype, copy=False)
else:
results = np.asanyarray(
[np.ndarray.astype(res, dtype, copy=False)])
# The first iteration is only to get sample results and from them
# compute some properties of the output dataset.
if dst is None:
kwargs['count'] = results.shape[0]
dst = rasterio.open(output, 'w', **kwargs)
work_windows.extend(
_chunk_output(dst.width,
dst.height,
dst.count,
np.dtype(dst.dtypes[0]).itemsize,
mem_limit=mem_limit))
# In subsequent iterations we write results.
else:
dst.write(results, window=window)
except snuggs.ExpressionError as err:
click.echo("Expression Error:")
click.echo(' %s' % err.text)
click.echo(' ' + ' ' * err.offset + "^")
click.echo(err)
raise click.Abort()
finally:
if dst:
dst.close()
for src in sources:
src.close()
def test_fillnodata_mask_ones(hole_in_ones):
"""when mask is all ones, image should be unmodified"""
mask = np.ones((5, 5))
result = fillnodata(hole_in_ones, mask)
assert (hole_in_ones == result).all()
