def calc(ctx, command, files, output, name, dtype, masked, creation_options):
"""A raster data calculator
Evaluates an expression using input datasets and writes the result
to a new dataset.
Command syntax is lisp-like. An expression consists of an operator
or function name and one or more strings, numbers, or expressions
enclosed in parentheses. Functions include ``read`` (gets a raster
array) and ``asarray`` (makes a 3-D array from 2-D arrays).
\b
* (read i) evaluates to the i-th input dataset (a 3-D array).
* (read i j) evaluates to the j-th band of the i-th dataset (a 2-D
array).
* (take foo j) evaluates to the j-th band of a dataset named foo (see
help on the --name option above).
* Standard numpy array operators (+, -, *, /) are available.
* When the final result is a list of arrays, a multi band output
file is written.
* When the final result is a single array, a single band output
file is written.
Example:
\b
$ rio calc "(+ 2 (* 0.95 (read 1)))" tests/data/RGB.byte.tif \\
> /tmp/out.tif
Produces a 3-band GeoTIFF with all values scaled by 0.95 and
incremented by 2.
\b
$ rio calc "(asarray (+ 125 (read 1)) (read 1) (read 1))" \\
> tests/data/shade.tif /tmp/out.tif
Produces a 3-band RGB GeoTIFF, with red levels incremented by 125,
from the single-band input.
"""
import numpy as np
verbosity = (ctx.obj and ctx.obj.get('verbosity')) or 1
logger = logging.getLogger('rio')
try:
with rasterio.drivers(CPL_DEBUG=verbosity > 2):
output, files = resolve_inout(files=files, output=output)
inputs = ([tuple(n.split('=')) for n in name] + [(None, n)
for n in files])
with rasterio.open(inputs[0][1]) as first:
kwargs = first.meta
kwargs.update(**creation_options)
kwargs['transform'] = kwargs.pop('affine')
dtype = dtype or first.meta['dtype']
kwargs['dtype'] = dtype
ctxkwds = {}
for i, (name, path) in enumerate(inputs):
with rasterio.open(path) as src:
# Using the class method instead of instance
# method. Latter raises
#
# TypeError: astype() got an unexpected keyword
# argument 'copy'
#
# possibly something to do with the instance being
# a masked array.
ctxkwds[name or '_i%d' % (i + 1)] = src.read(masked=masked)
# Extend snuggs.
snuggs.func_map['read'] = read_array
snuggs.func_map['band'] = lambda d, i: get_bands(inputs, d, i)
snuggs.func_map['bands'] = lambda d: get_bands(inputs, d)
snuggs.func_map['fillnodata'] = lambda *args: fillnodata(*args)
snuggs.func_map['sieve'] = lambda *args: sieve(*args)
res = snuggs.eval(command, **ctxkwds)
if (isinstance(res, np.ma.core.MaskedArray)
and tuple(LooseVersion(np.__version__).version) <
(1, 9, 0)):
res = res.filled(kwargs['nodata'])
if len(res.shape) == 3:
results = np.ndarray.astype(res, dtype, copy=False)
else:
results = np.asanyarray(
[np.ndarray.astype(res, dtype, copy=False)])
kwargs['count'] = results.shape[0]
with rasterio.open(output, 'w', **kwargs) as dst:
dst.write(results)
except snuggs.ExpressionError as err:
click.echo("Expression Error:")
click.echo(' %s' % err.text)
click.echo(' ' + ' ' * err.offset + "^")
click.echo(err)
raise click.Abort()
def test_partial():
result = snuggs.eval('((partial * 2) 2)')
assert result == 4
def test_arr_lookup_kwarg_order(ones):
kwargs = OrderedDict(
(('foo', ones), ('bar', 2.0 * ones), ('a', 3.0 * ones)))
r = snuggs.eval('(read 1)', **kwargs)
assert list(r.flatten()) == [1, 1, 1, 1]
def test_int_arr_expr_by_name(ones):
result = snuggs.eval('(+ (read 1) 1.5)', foo=ones)
assert list(result.flatten()) == [2.5, 2.5, 2.5, 2.5]
def test_negative_decimal():
"""Negative decimals parse correctly"""
assert snuggs.eval("(< -0.9 0)")
def test_int_real_expr():
assert snuggs.eval('(+ 2 1.1)') == 3.1
def test_int_expr():
assert snuggs.eval('(+ 1 2)') == 3
def test_missing_func2():
with pytest.raises(SyntaxError) as excinfo:
snuggs.eval("(# 1 2)")
assert excinfo.value.lineno == 1
assert excinfo.value.offset == 2
def test_masked_arr():
foo = numpy.ma.masked_equal(numpy.array([0, 0, 0, 1], dtype='uint8'), 0)
r = snuggs.eval('(+ foo 1)', foo=foo)
assert list(r.data.flatten()) == [0, 0, 0, 2]
assert list(r.flatten()) == [numpy.ma.masked, numpy.ma.masked, numpy.ma.masked, 2]
def test_missing_closing_paren():
with pytest.raises(SyntaxError) as excinfo:
result = snuggs.eval("(+ 1 2")
assert excinfo.value.lineno == 1
assert excinfo.value.offset == 7
assert str(excinfo.value) == 'Expected ")"'
def test_nil():
assert snuggs.eval('(== nil nil)')
assert not snuggs.eval('(== 1 nil)')
assert not snuggs.eval('(== nil 1)')
assert snuggs.eval('(!= 1 nil)')
assert snuggs.eval('(!= nil 1)')
def test_map_asarray():
result = snuggs.eval('(asarray (map (partial * 2) (asarray 1 2 3)))')
assert list(result) == [2, 4, 6]
def test_map_func():
result = snuggs.eval('(map sqrt (asarray 1 4 9))')
assert list(result) == [1, 2, 3]
def test_nil():
assert snuggs.eval('(== nil nil)')
assert not snuggs.eval('(== 1 nil)')
assert not snuggs.eval('(== nil 1)')
assert snuggs.eval('(!= 1 nil)')
assert snuggs.eval('(!= nil 1)')
def test_missing_func2():
with pytest.raises(SyntaxError) as excinfo:
result = snuggs.eval("(# 1 2)")
assert excinfo.value.lineno == 1
assert excinfo.value.offset == 2
assert str(excinfo.value) == "expected a function or operator"
def test_missing_closing_paren():
with pytest.raises(SyntaxError) as excinfo:
snuggs.eval("(+ 1 2")
assert excinfo.value.lineno == 1
assert excinfo.value.offset == 7
def test_undefined_var():
with pytest.raises(SyntaxError) as excinfo:
result = snuggs.eval("(+ 1 bogus)")
assert excinfo.value.lineno == 1
assert excinfo.value.offset == 6
assert str(excinfo.value) == "name 'bogus' is not defined"
def test_bogus_higher_order_func():
with pytest.raises(SyntaxError) as excinfo:
snuggs.eval("((bogus * 2) 2)")
assert excinfo.value.lineno == 1
assert excinfo.value.offset == 3
def test_bogus_higher_order_func():
with pytest.raises(SyntaxError) as excinfo:
result = snuggs.eval("((bogus * 2) 2)")
assert excinfo.value.lineno == 1
assert excinfo.value.offset == 3
assert str(excinfo.value) == "expected a function or operator"
def test_int_mult_expr():
assert snuggs.eval('(+ 1 2 3)') == 6
def test_type_error():
with pytest.raises(TypeError) as excinfo:
result = snuggs.eval("(+ 1 'bogus')")
def test_arr_var(ones):
r = snuggs.eval('(+ foo 0)', foo=ones)
assert list(r.flatten()) == [1, 1, 1, 1]
def test_real_expr():
assert round(snuggs.eval('(* 0.1 0.2)'), 3) == 0.02
def test_arr_take(ones):
r = snuggs.eval('(take foo 1)', foo=ones)
assert list(r.flatten()) == [1, 1]
r = snuggs.eval('(take foo 2)', foo=ones)
assert list(r.flatten()) == [1, 1]
def test_int_real_expr():
assert snuggs.eval('(+ 2 1.1)') == 3.1
def image_composite(inputs, algo, output, oformat, vza, mask_band, mask_val):
""" Create image composites based on some criteria
Output image composites retain original values from input images that meet
a certain criteria. For example, in a maximum NDVI composite with 10 input
images, all bands for a given pixel will contain the band values from the
input raster that had the highest NDVI value.
Users can choose from a set of predefined compositing algorithms or may
specify an Snuggs S-expression that defines the compositing criteria.
Normalized Differenced indexes can be computed using "(normdiff a b)" for
the Normalized Difference between "a" and "b" (or "nir" and "red").
See https://github.com/mapbox/snuggs for more information on Snuggs
expressions.
The indexes for common optical bands (e.g., red, nir, blue) within the
input rasters are included as optional arguments and are indexed in
wavelength sequential order. You may need to overwrite the default indexes
of bands used in a given S-expression with the correct band index.
Additional bands may be identified and indexed using the
'--band NAME=INDEX' option.
Currently, input images must be "stacked", meaning that they contain the
same bands and are the same shape and extent.
Example:
1. Create a composite based on maximum NDVI
Use the built-in maxNDVI algorithm:
\b
$ image_composite.py --algo maxNDVI image1.gtif image2.gtif image3.gtif
composite_maxNDVI.gtif
or with S-expression:
\b
$ image_composite.py --expr '(max (/ (- nir red) (+ nir red)))'
image1.gtif image2.gtif image3.gtif composite_maxNDVI.gtif
or with S-expressions using the normdiff shortcut:
\b
$ image_composite.py --expr '(max (normdiff nir red))'
image1.gtif image2.gtif image3.gtif composite_maxNDVI.gtif
2. Create a composite based on median EVI (not recommended)
With S-expression:
\b
$ evi='(median (/ (- nir red) (+ (- (+ nir (* 6 red)) (* 7.5 blue)) 1)))'
$ image_composite.py --expr "$evi" image1.gtif image2.gtif image3.gtif
composite_medianEVI.gtif
3. Create a composite based on median NBR
With S-expression:
\b
$ image_composite.py --expr '(median (normdiff nir sswir))'
image1.gtif image2.gtif image3.gtif composite_maxNBR.gtif
"""
verbose = True
if verbose:
logger.setLevel(logging.DEBUG)
elif quiet:
logger.setLevel(logging.ERROR)
expr = _ALGO[algo]
if algo is not None:
logger.debug('Using predefined algorithm: {}'.format(algo))
expr = _ALGO[algo]
# Setup band keywords
_bands = {'vza': vza}
# Find only the band names and indexes required for the composite criteria
crit_indices = {k: v - 1 for k, v in _bands.iteritems() if k in expr}
# Enhance snuggs expressions to return index of value matching function
snuggs.func_map['max'] = lambda a: np.argmax(a, axis=0)
snuggs.func_map['min'] = lambda a: np.argmin(a, axis=0)
snuggs.func_map['median'] = lambda a: np.argmin(
np.abs(a - np.median(a, axis=0)), axis=0)
snuggs.func_map['normdiff'] = lambda a, b: snuggs.eval(
'(/ (- a b) (+ a b))', **{
'a': a,
'b': b
})
with rasterio.drivers():
# Read in the first image to fetch metadata
with rasterio.open(inputs[0]) as first:
meta = first.meta
if 'transform' in meta:
meta.pop('transform') # remove transform since deprecated
meta.update(driver=oformat)
if len(set(first.block_shapes)) != 1:
click.echo('Cannot process input files - '
'All bands must have same block shapes')
raise click.Abort()
block_nrow, block_ncol = first.block_shapes[0]
windows = first.block_windows(1)
n_windows = math.ceil(meta['height'] / block_nrow * meta['width'] /
block_ncol)
# Ensure mask_band exists, if specified
if mask_band:
if mask_band <= meta['count'] and mask_band > 0:
mask_band -= 1
else:
click.echo('Mask band does not exist in INPUT images')
raise click.Abort()
# Initialize output data and create composite
with rasterio.open(output, 'w', **meta) as dst:
# Process by block
dat = np.ma.empty(
(len(inputs), meta['count'], block_nrow, block_ncol),
dtype=np.dtype(meta['dtype']))
mi, mj = np.meshgrid(np.arange(block_nrow),
np.arange(block_ncol),
indexing='ij')
# Open all source files one time
srcs = [rasterio.open(fname) for fname in inputs]
logger.debug('Processing blocks')
if _has_progressbar:
widgets = [
progressbar.Percentage(),
progressbar.BouncingBar(
marker=progressbar.RotatingMarker())
]
pbar = progressbar.ProgressBar(widgets=widgets).start()
for i, (idx, window) in enumerate(windows):
# Update dat and mi, mj only if window changes
nrow = window[0][1] - window[0][0]
ncol = window[1][1] - window[1][0]
if dat.shape[-2] != nrow or dat.shape[-1] != ncol:
dat = np.ma.empty((len(inputs), meta['count'], nrow, ncol),
dtype=np.dtype(meta['dtype']))
mi, mj = np.meshgrid(np.arange(nrow),
np.arange(ncol),
indexing='ij')
for j, src in enumerate(srcs):
dat[j, ...] = src.read(masked=True, window=window)
# Mask values matching mask_vals if mask_band
if mask_band and mask_val:
dat[j, ...].mask = np.logical_or(
dat[j, ...].mask,
np.in1d(
dat[j, mask_band, ...],
mask_val,
).reshape(dat.shape[-2], dat.shape[-1]))
# Find indices of files for composite
crit = {k: dat[:, v, ...] for k, v in crit_indices.iteritems()}
crit_idx = snuggs.eval(expr, **crit)
# Create output composite
# Use np.rollaxis to get (nimage, nrow, ncol, nband) shape
composite = np.rollaxis(dat, 1, 4)[crit_idx, mi, mj]
# Write out
for i_b in range(composite.shape[-1]):
dst.write(composite[:, :, i_b],
indexes=i_b + 1,
window=window)
if _has_progressbar:
pbar.update(int((i + 1) / n_windows * 100))
def test_real_int_expr():
assert snuggs.eval('(+ 1.1 2)') == 3.1
def calc(ctx, command, files, name, dtype):
"""A raster data calculator
Evaluates an expression using input datasets and writes the result
to a new dataset.
Command syntax is lisp-like. An expression consists of an operator
or function name and one or more strings, numbers, or expressions
enclosed in parentheses. Functions include ``read`` (gets a raster
array) and ``asarray`` (makes a 3-D array from 2-D arrays).
\b
* (read i) evaluates to the i-th input dataset (a 3-D array).
* (read i j) evaluates to the j-th band of the i-th dataset (a 2-D
array).
* (take foo j) evaluates to the j-th band of a dataset named foo (see
help on the --name option above).
* Standard numpy array operators (+, -, *, /) are available.
* When the final result is a list of arrays, a multi band output
file is written.
* When the final result is a single array, a single band output
file is written.
Example:
\b
$ rio calc "(+ 2 (* 0.95 (read 1)))" tests/data/RGB.byte.tif \\
> /tmp/out.tif
Produces a 3-band GeoTIFF with all values scaled by 0.95 and
incremented by 2.
\b
$ rio calc "(asarray (+ 125 (read 1)) (read 1) (read 1))" \\
> tests/data/shade.tif /tmp/out.tif
Produces a 3-band RGB GeoTIFF, with red levels incremented by 125,
from the single-band input.
"""
import numpy as np
verbosity = (ctx.obj and ctx.obj.get('verbosity')) or 1
logger = logging.getLogger('rio')
try:
with rasterio.drivers(CPL_DEBUG=verbosity > 2):
output = files[-1]
inputs = ([tuple(n.split('=')) for n in name] +
[(None, n) for n in files[:-1]])
with rasterio.open(inputs[0][1]) as first:
kwargs = first.meta
kwargs['transform'] = kwargs.pop('affine')
dtype = dtype or first.meta['dtype']
kwargs['dtype'] = dtype
ctxkwds = {}
for i, (name, path) in enumerate(inputs):
with rasterio.open(path) as src:
# Using the class method instead of instance
# method. Latter raises
#
# TypeError: astype() got an unexpected keyword
# argument 'copy'
#
# possibly something to do with the instance being
# a masked array.
ctxkwds[name or '_i%d' % (i+1)] = src.read()
# Extend snuggs.
snuggs.func_map['read'] = read_array
snuggs.func_map['band'] = lambda d, i: get_bands(inputs, d, i)
snuggs.func_map['bands'] = lambda d: get_bands(inputs, d)
snuggs.func_map['fillnodata'] = lambda *args: fillnodata(*args)
snuggs.func_map['sieve'] = lambda *args: sieve(*args)
res = snuggs.eval(command, **ctxkwds)
if len(res.shape) == 3:
results = np.ndarray.astype(res, dtype, copy=False)
else:
results = np.asanyarray(
[np.ndarray.astype(res, dtype, copy=False)])
kwargs['count'] = results.shape[0]
with rasterio.open(output, 'w', **kwargs) as dst:
dst.write(results)
sys.exit(0)
except snuggs.ExpressionError as err:
click.echo("Expression Error:")
click.echo(' %s' % err.text)
click.echo(' ' + ' ' * err.offset + "^")
click.echo(err)
sys.exit(1)
except Exception as err:
t, v, tb = sys.exc_info()
for line in traceback.format_exception_only(t, v):
click.echo(line, nl=False)
sys.exit(1)
def test_arr_var(ones):
r = snuggs.eval('(+ foo 0)', foo=ones)
assert list(r.flatten()) == [1, 1, 1, 1]
def test_multi_operator_array(ones):
result = snuggs.eval('(+ ones (/ ones 1 0.5) (* ones 1 3))', ones=ones)
assert list(result.flatten()) == [6.0] * 4
def test_arr_lookup_2(ones):
r = snuggs.eval('(read 1 1)', foo=ones)
assert list(r.flatten()) == [1, 1]
def test_masked_arr():
foo = numpy.ma.masked_equal(numpy.array([0, 0, 0, 1], dtype='uint8'), 0)
r = snuggs.eval('(+ foo 1)', foo=foo)
assert list(r.data.flatten()) == [0, 0, 0, 2]
assert list(
r.flatten()) == [numpy.ma.masked, numpy.ma.masked, numpy.ma.masked, 2]
def test_arr_take(ones):
r = snuggs.eval('(take foo 1)', foo=ones)
assert list(r.flatten()) == [1, 1]
r = snuggs.eval('(take foo 2)', foo=ones)
assert list(r.flatten()) == [1, 1]
def test_missing_func():
with pytest.raises(SyntaxError) as excinfo:
snuggs.eval("(0 1 2)")
assert excinfo.value.lineno == 1
assert excinfo.value.offset == 2
assert str(excinfo.value) == "'0' is not a function or operator"
def test_int_arr_expr(ones):
result = snuggs.eval('(+ foo 1)', foo=ones)
assert list(result.flatten()) == [2, 2, 2, 2]
def test_undefined_var():
with pytest.raises(SyntaxError) as excinfo:
snuggs.eval("(+ 1 bogus)")
assert excinfo.value.lineno == 1
assert excinfo.value.offset == 6
assert str(excinfo.value) == "name 'bogus' is not defined"
def test_int_arr_read(ones):
result = snuggs.eval('(+ (read 1 1) 1.5)', foo=ones)
assert list(result.flatten()) == [2.5, 2.5]
def test_type_error():
with pytest.raises(TypeError):
snuggs.eval("(+ 1 'bogus')")
def test_list(ones):
result = snuggs.eval(
'(asarray (take foo 1) (take foo 1) (take bar 1) (take bar 1))',
foo=ones, bar=ones)
assert list(result.flatten()) == [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]
def test_int_expr():
assert snuggs.eval('(+ 1 2)') == 3
def test_eq(ones):
ones[0][0] = 2
result = snuggs.eval('(== foo 1)', foo=ones)
assert list(result.flatten()) == [False, True, True, True]
def test_real_expr():
assert round(snuggs.eval('(* 0.1 0.2)'), 3) == 0.02
def test_and(truetrue, truefalse):
result = snuggs.eval(
'(& foo bar)', foo=truetrue, bar=truefalse)
assert list(result.flatten()) == [True, False]
def test_real_int_expr():
assert snuggs.eval('(+ 1.1 2)') == 3.1
def test_ones_like(truefalse):
result = snuggs.eval("(ones_like foo 'uint8')", foo=truefalse)
assert list(result.flatten()) == [1.0, 1.0]
def test_arr_var_long(ones):
r = snuggs.eval('(+ FOO_BAR_42 0)', FOO_BAR_42=ones)
assert list(r.flatten()) == [1, 1, 1, 1]
def test_full_like(truefalse):
result = snuggs.eval("(full_like foo 3.14 'float64')", foo=truefalse)
assert list(result.flatten()) == [3.14, 3.14]
result = snuggs.eval('(full_like foo 3.14 "float64")', foo=truefalse)
assert list(result.flatten()) == [3.14, 3.14]
def test_arr_lookup_2(ones):
r = snuggs.eval('(read 1 1)', foo=ones)
assert list(r.flatten()) == [1, 1]
def test_full_like(truefalse):
result = snuggs.eval("(full_like foo 3.14 'float64')", foo=truefalse)
assert list(result.flatten()) == [3.14, 3.14]
result = snuggs.eval('(full_like foo 3.14 "float64")', foo=truefalse)
assert list(result.flatten()) == [3.14, 3.14]
def test_int_arr_expr(ones):
result = snuggs.eval('(+ foo 1)', foo=ones)
assert list(result.flatten()) == [2, 2, 2, 2]
def test_ufunc(truetrue, truefalse):
result = snuggs.eval('(where (& foo bar) 1 0)',
foo=truetrue,
bar=truefalse)
assert list(result.flatten()) == [1.0, 0.0]
def calc(ctx, command, files, output, name, dtype, masked, overwrite, mem_limit, creation_options):
"""A raster data calculator
Evaluates an expression using input datasets and writes the result
to a new dataset.
Command syntax is lisp-like. An expression consists of an operator
or function name and one or more strings, numbers, or expressions
enclosed in parentheses. Functions include ``read`` (gets a raster
array) and ``asarray`` (makes a 3-D array from 2-D arrays).
\b
* (read i) evaluates to the i-th input dataset (a 3-D array).
* (read i j) evaluates to the j-th band of the i-th dataset (a
2-D array).
* (take foo j) evaluates to the j-th band of a dataset named foo
(see help on the --name option above).
* Standard numpy array operators (+, -, *, /) are available.
* When the final result is a list of arrays, a multiple band
output file is written.
* When the final result is a single array, a single band output
file is written.
Example:
\b
$ rio calc "(+ 2 (* 0.95 (read 1)))" tests/data/RGB.byte.tif \\
> /tmp/out.tif
The command above produces a 3-band GeoTIFF with all values scaled
by 0.95 and incremented by 2.
\b
$ rio calc "(asarray (+ 125 (read 1)) (read 1) (read 1))" \\
> tests/data/shade.tif /tmp/out.tif
The command above produces a 3-band RGB GeoTIFF, with red levels
incremented by 125, from the single-band input.
The maximum amount of memory used to perform caculations defaults to
64 MB. This number can be increased to improve speed of calculation.
"""
import numpy as np
try:
with ctx.obj['env']:
output, files = resolve_inout(files=files, output=output,
overwrite=overwrite)
inputs = ([tuple(n.split('=')) for n in name] +
[(None, n) for n in files])
sources = [rasterio.open(path) for name, path in inputs]
first = sources[0]
kwargs = first.profile
kwargs.update(**creation_options)
dtype = dtype or first.meta['dtype']
kwargs['dtype'] = dtype
# Extend snuggs.
snuggs.func_map['read'] = _read_array
snuggs.func_map['band'] = lambda d, i: _get_bands(inputs, sources, d, i)
snuggs.func_map['bands'] = lambda d: _get_bands(inputs, sources, d)
snuggs.func_map['fillnodata'] = lambda *args: fillnodata(*args)
snuggs.func_map['sieve'] = lambda *args: sieve(*args)
dst = None
# The windows iterator is initialized with a single sample.
# The actual work windows will be added in the second
# iteration of the loop.
work_windows = [(None, Window(0, 0, 16, 16))]
for ij, window in work_windows:
ctxkwds = OrderedDict()
for i, ((name, path), src) in enumerate(zip(inputs, sources)):
# Using the class method instead of instance
# method. Latter raises
#
# TypeError: astype() got an unexpected keyword
# argument 'copy'
#
# possibly something to do with the instance being
# a masked array.
ctxkwds[name or '_i%d' % (i + 1)] = src.read(masked=masked, window=window)
res = snuggs.eval(command, **ctxkwds)
if (isinstance(res, np.ma.core.MaskedArray) and (
tuple(LooseVersion(np.__version__).version) < (1, 9) or
tuple(LooseVersion(np.__version__).version) > (1, 10))):
res = res.filled(kwargs['nodata'])
if len(res.shape) == 3:
results = np.ndarray.astype(res, dtype, copy=False)
else:
results = np.asanyarray(
[np.ndarray.astype(res, dtype, copy=False)])
# The first iteration is only to get sample results and from them
# compute some properties of the output dataset.
if dst is None:
kwargs['count'] = results.shape[0]
dst = rasterio.open(output, 'w', **kwargs)
work_windows.extend(_chunk_output(dst.width, dst.height, dst.count, np.dtype(dst.dtypes[0]).itemsize, mem_limit=mem_limit))
# In subsequent iterations we write results.
else:
dst.write(results, window=window)
except snuggs.ExpressionError as err:
click.echo("Expression Error:")
click.echo(' %s' % err.text)
click.echo(' ' + ' ' * err.offset + "^")
click.echo(err)
raise click.Abort()
finally:
if dst:
dst.close()
for src in sources:
src.close()
def test_partial():
result = snuggs.eval('((partial * 2) 2)')
assert result == 4
def image_composite(inputs, algo, expr, output, oformat, creation_options,
blue, green, red, nir, fswir, sswir, band,
mask_band, mask_val,
verbose, quiet):
""" Create image composites based on some criteria
Output image composites retain original values from input images that meet
a certain criteria. For example, in a maximum NDVI composite with 10 input
images, all bands for a given pixel will contain the band values from the
input raster that had the highest NDVI value.
Users can choose from a set of predefined compositing algorithms or may
specify an Snuggs S-expression that defines the compositing criteria.
Normalized Differenced indexes can be computed using "(normdiff a b)" for
the Normalized Difference between "a" and "b" (or "nir" and "red").
See https://github.com/mapbox/snuggs for more information on Snuggs
expressions.
The indexes for common optical bands (e.g., red, nir, blue) within the
input rasters are included as optional arguments and are indexed in
wavelength sequential order. You may need to overwrite the default indexes
of bands used in a given S-expression with the correct band index.
Additional bands may be identified and indexed using the
'--band NAME=INDEX' option.
Currently, input images must be "stacked", meaning that they contain the
same bands and are the same shape and extent.
Example:
1. Create a composite based on maximum NDVI
Use the built-in maxNDVI algorithm:
\b
$ image_composite.py --algo maxNDVI image1.gtif image2.gtif image3.gtif
composite_maxNDVI.gtif
or with S-expression:
\b
$ image_composite.py --expr '(max (/ (- nir red) (+ nir red)))'
image1.gtif image2.gtif image3.gtif composite_maxNDVI.gtif
or with S-expressions using the normdiff shortcut:
\b
$ image_composite.py --expr '(max (normdiff nir red))'
image1.gtif image2.gtif image3.gtif composite_maxNDVI.gtif
2. Create a composite based on median EVI (not recommended)
With S-expression:
\b
$ evi='(median (/ (- nir red) (+ (- (+ nir (* 6 red)) (* 7.5 blue)) 1)))'
$ image_composite.py --expr "$evi" image1.gtif image2.gtif image3.gtif
composite_medianEVI.gtif
3. Create a composite based on median NBR
With S-expression:
\b
$ image_composite.py --expr '(median (normdiff nir sswir))'
image1.gtif image2.gtif image3.gtif composite_maxNBR.gtif
"""
if verbose:
logger.setLevel(logging.DEBUG)
elif quiet:
logger.setLevel(logging.ERROR)
# Prefer built-in algorithms to expressions if both are specified
if not algo and not expr:
raise click.UsageError('Error: must specify either --algo or --expr')
elif algo is not None and expr is not None:
logger.warning('Predefined algorithm and expression both defined. '
'Composite will be generated with predefined algorithm')
expr = _ALGO[algo]
elif algo is not None:
logger.debug('Using predefined algorithm: {}'.format(algo))
expr = _ALGO[algo]
if not quiet:
click.echo('Compositing criteria S-expression: "{}"'.format(expr))
# Setup band keywords
_bands = {'blue': blue, 'green': green, 'red': red,
'nir': nir, 'fswir': fswir, 'sswir': sswir}
# Parse any additional, user specified bands
if band:
for k, v in band.iteritems():
try:
_bands.update({k: int(v)})
except ValueError:
raise click.BadParameter(
'Value specified as KEY=VAL pair in --band must be an int')
# Find only the band names and indexes required for the composite criteria
crit_indices = {k: v - 1 for k, v in _bands.iteritems() if k in expr}
# Enhance snuggs expressions to return index of value matching function
snuggs.func_map['max'] = lambda a: np.argmax(a, axis=0)
snuggs.func_map['min'] = lambda a: np.argmin(a, axis=0)
snuggs.func_map['median'] = lambda a: np.argmin(
np.abs(a - np.median(a, axis=0)), axis=0)
snuggs.func_map['normdiff'] = lambda a, b: snuggs.eval(
'(/ (- a b) (+ a b))', **{'a':a, 'b':b})
with rasterio.drivers():
# Read in the first image to fetch metadata
with rasterio.open(inputs[0]) as first:
meta = first.meta
if 'transform' in meta:
meta.pop('transform') # remove transform since deprecated
meta.update(driver=oformat)
meta.update(**creation_options)
if len(set(first.block_shapes)) != 1:
click.echo('Cannot process input files - '
'All bands must have same block shapes')
raise click.Abort()
block_nrow, block_ncol = first.block_shapes[0]
windows = first.block_windows(1)
n_windows = math.ceil(meta['height'] / block_nrow *
meta['width'] / block_ncol)
# Ensure mask_band exists, if specified
if mask_band:
if mask_band <= meta['count'] and mask_band > 0:
mask_band -= 1
else:
click.echo('Mask band does not exist in INPUT images')
raise click.Abort()
# Initialize output data and create composite
with rasterio.open(output, 'w', **meta) as dst:
# Process by block
dat = np.ma.empty((len(inputs), meta['count'],
block_nrow, block_ncol),
dtype=np.dtype(meta['dtype']))
mi, mj = np.meshgrid(np.arange(block_nrow), np.arange(block_ncol),
indexing='ij')
# Open all source files one time
srcs = [rasterio.open(fname) for fname in inputs]
logger.debug('Processing blocks')
if _has_progressbar and not quiet:
widgets = [
progressbar.Percentage(),
progressbar.BouncingBar(
marker=progressbar.RotatingMarker())
]
pbar = progressbar.ProgressBar(widgets=widgets).start()
for i, (idx, window) in enumerate(windows):
# Update dat and mi, mj only if window changes
nrow = window[0][1] - window[0][0]
ncol = window[1][1] - window[1][0]
if dat.shape[-2] != nrow or dat.shape[-1] != ncol:
dat = np.ma.empty((len(inputs), meta['count'],
nrow, ncol),
dtype=np.dtype(meta['dtype']))
mi, mj = np.meshgrid(np.arange(nrow), np.arange(ncol),
indexing='ij')
for j, src in enumerate(srcs):
dat[j, ...] = src.read(masked=True, window=window)
# Mask values matching mask_vals if mask_band
if mask_band and mask_val:
dat[j, ...].mask = np.logical_or(
dat[j, ...].mask,
np.in1d(dat[j, mask_band, ...], mask_val,).reshape(
dat.shape[-2], dat.shape[-1])
)
# Find indices of files for composite
crit = {k: dat[:, v, ...] for k, v in crit_indices.iteritems()}
crit_idx = snuggs.eval(expr, **crit)
# Create output composite
# Use np.rollaxis to get (nimage, nrow, ncol, nband) shape
composite = np.rollaxis(dat, 1, 4)[crit_idx, mi, mj]
# Write out
for i_b in range(composite.shape[-1]):
dst.write(composite[:, :, i_b], indexes=i_b + 1,
window=window)
if not quiet and _has_progressbar:
pbar.update(int((i + 1) / n_windows * 100))
def test_map_func():
result = snuggs.eval('(map sqrt (asarray 1 4 9))')
assert list(result) == [1, 2, 3]
def calc(ctx, command, files, output, name, dtype, masked, overwrite,
creation_options):
"""A raster data calculator
Evaluates an expression using input datasets and writes the result
to a new dataset.
Command syntax is lisp-like. An expression consists of an operator
or function name and one or more strings, numbers, or expressions
enclosed in parentheses. Functions include ``read`` (gets a raster
array) and ``asarray`` (makes a 3-D array from 2-D arrays).
\b
* (read i) evaluates to the i-th input dataset (a 3-D array).
* (read i j) evaluates to the j-th band of the i-th dataset (a 2-D
array).
* (take foo j) evaluates to the j-th band of a dataset named foo (see
help on the --name option above).
* Standard numpy array operators (+, -, *, /) are available.
* When the final result is a list of arrays, a multi band output
file is written.
* When the final result is a single array, a single band output
file is written.
Example:
\b
$ rio calc "(+ 2 (* 0.95 (read 1)))" tests/data/RGB.byte.tif \\
> /tmp/out.tif
Produces a 3-band GeoTIFF with all values scaled by 0.95 and
incremented by 2.
\b
$ rio calc "(asarray (+ 125 (read 1)) (read 1) (read 1))" \\
> tests/data/shade.tif /tmp/out.tif
Produces a 3-band RGB GeoTIFF, with red levels incremented by 125,
from the single-band input.
"""
import numpy as np
try:
with ctx.obj['env']:
output, files = resolve_inout(files=files, output=output,
overwrite=overwrite)
inputs = ([tuple(n.split('=')) for n in name] +
[(None, n) for n in files])
with rasterio.open(inputs[0][1]) as first:
kwargs = first.meta
kwargs.update(**creation_options)
dtype = dtype or first.meta['dtype']
kwargs['dtype'] = dtype
ctxkwds = OrderedDict()
for i, (name, path) in enumerate(inputs):
with rasterio.open(path) as src:
# Using the class method instead of instance
# method. Latter raises
#
# TypeError: astype() got an unexpected keyword
# argument 'copy'
#
# possibly something to do with the instance being
# a masked array.
ctxkwds[name or '_i%d' % (i + 1)] = src.read(masked=masked)
# Extend snuggs.
snuggs.func_map['read'] = read_array
snuggs.func_map['band'] = lambda d, i: get_bands(inputs, d, i)
snuggs.func_map['bands'] = lambda d: get_bands(inputs, d)
snuggs.func_map['fillnodata'] = lambda *args: fillnodata(*args)
snuggs.func_map['sieve'] = lambda *args: sieve(*args)
res = snuggs.eval(command, ctxkwds)
if (isinstance(res, np.ma.core.MaskedArray) and (
tuple(LooseVersion(np.__version__).version) < (1, 9) or
tuple(LooseVersion(np.__version__).version) > (1, 10))):
res = res.filled(kwargs['nodata'])
if len(res.shape) == 3:
results = np.ndarray.astype(res, dtype, copy=False)
else:
results = np.asanyarray(
[np.ndarray.astype(res, dtype, copy=False)])
kwargs['count'] = results.shape[0]
with rasterio.open(output, 'w', **kwargs) as dst:
dst.write(results)
except snuggs.ExpressionError as err:
click.echo("Expression Error:")
click.echo(' %s' % err.text)
click.echo(' ' + ' ' * err.offset + "^")
click.echo(err)
raise click.Abort()
def test_map_asarray():
result = snuggs.eval('(asarray (map (partial * 2) (asarray 1 2 3)))')
assert list(result) == [2, 4, 6]
def calc(ctx, command, files, output, name, dtype, masked, overwrite,
mem_limit, creation_options):
"""A raster data calculator
Evaluates an expression using input datasets and writes the result
to a new dataset.
Command syntax is lisp-like. An expression consists of an operator
or function name and one or more strings, numbers, or expressions
enclosed in parentheses. Functions include ``read`` (gets a raster
array) and ``asarray`` (makes a 3-D array from 2-D arrays).
\b
* (read i) evaluates to the i-th input dataset (a 3-D array).
* (read i j) evaluates to the j-th band of the i-th dataset (a
2-D array).
* (take foo j) evaluates to the j-th band of a dataset named foo
(see help on the --name option above).
* Standard numpy array operators (+, -, *, /) are available.
* When the final result is a list of arrays, a multiple band
output file is written.
* When the final result is a single array, a single band output
file is written.
Example:
\b
$ rio calc "(+ 2 (* 0.95 (read 1)))" tests/data/RGB.byte.tif \\
> /tmp/out.tif
The command above produces a 3-band GeoTIFF with all values scaled
by 0.95 and incremented by 2.
\b
$ rio calc "(asarray (+ 125 (read 1)) (read 1) (read 1))" \\
> tests/data/shade.tif /tmp/out.tif
The command above produces a 3-band RGB GeoTIFF, with red levels
incremented by 125, from the single-band input.
The maximum amount of memory used to perform caculations defaults to
64 MB. This number can be increased to improve speed of calculation.
"""
import numpy as np
dst = None
sources = []
try:
with ctx.obj['env']:
output, files = resolve_inout(files=files,
output=output,
overwrite=overwrite)
inputs = ([tuple(n.split('=')) for n in name] + [(None, n)
for n in files])
sources = [rasterio.open(path) for name, path in inputs]
first = sources[0]
kwargs = first.profile
kwargs.update(**creation_options)
dtype = dtype or first.meta['dtype']
kwargs['dtype'] = dtype
# Extend snuggs.
snuggs.func_map['read'] = _read_array
snuggs.func_map['band'] = lambda d, i: _get_bands(
inputs, sources, d, i)
snuggs.func_map['bands'] = lambda d: _get_bands(inputs, sources, d)
snuggs.func_map['fillnodata'] = lambda *args: fillnodata(*args)
snuggs.func_map['sieve'] = lambda *args: sieve(*args)
# The windows iterator is initialized with a single sample.
# The actual work windows will be added in the second
# iteration of the loop.
work_windows = [(None, Window(0, 0, 16, 16))]
for ij, window in work_windows:
ctxkwds = OrderedDict()
for i, ((name, path), src) in enumerate(zip(inputs, sources)):
# Using the class method instead of instance
# method. Latter raises
#
# TypeError: astype() got an unexpected keyword
# argument 'copy'
#
# possibly something to do with the instance being
# a masked array.
ctxkwds[name or '_i%d' % (i + 1)] = src.read(masked=masked,
window=window)
res = snuggs.eval(command, **ctxkwds)
if (isinstance(res, np.ma.core.MaskedArray) and
(tuple(LooseVersion(np.__version__).version) <
(1, 9) or tuple(LooseVersion(np.__version__).version) >
(1, 10))):
res = res.filled(kwargs['nodata'])
if len(res.shape) == 3:
results = np.ndarray.astype(res, dtype, copy=False)
else:
results = np.asanyarray(
[np.ndarray.astype(res, dtype, copy=False)])
# The first iteration is only to get sample results and from them
# compute some properties of the output dataset.
if dst is None:
kwargs['count'] = results.shape[0]
dst = rasterio.open(output, 'w', **kwargs)
work_windows.extend(
_chunk_output(dst.width,
dst.height,
dst.count,
np.dtype(dst.dtypes[0]).itemsize,
mem_limit=mem_limit))
# In subsequent iterations we write results.
else:
dst.write(results, window=window)
except snuggs.ExpressionError as err:
click.echo("Expression Error:")
click.echo(' %s' % err.text)
click.echo(' ' + ' ' * err.offset + "^")
click.echo(err)
raise click.Abort()
finally:
if dst:
dst.close()
for src in sources:
src.close()
def test_ufunc(truetrue, truefalse):
result = snuggs.eval(
'(where (& foo bar) 1 0)', foo=truetrue, bar=truefalse)
assert list(result.flatten()) == [1.0, 0.0]
