def band_data_to_image(band_data, colormap):
"""
Creates an python image from pixel values of a GDALRaster.
The input is a dictionary that maps pixel values to RGBA UInt8 colors.
"""
# Get data as 1D array
dat = band_data.ravel()
# Create zeros array
rgba = numpy.zeros((dat.shape[0], 4), dtype='uint8')
# Replace matched rows with colors
stats = {}
for key, color in colormap.items():
orig_key = key
try:
# Try to use the key as number directly
key = float(key)
selector = dat == key
except ValueError:
# Otherwise use it as numpy expression directly
parser = FormulaParser()
selector = parser.evaluate({'x': dat}, key)
# If masked, use mask to filter values additional to formula values
if numpy.ma.is_masked(selector):
selector.fill_value = False
rgba[selector.filled() == 1] = color
# Compress for getting statistics
selector = selector.compressed()
else:
rgba[selector] = color
# Track pixel statistics for this tile
stats[orig_key] = int(numpy.sum(selector))
# Reshape array to image size
rgba = rgba.reshape(band_data.shape[0], band_data.shape[1], 4)
# Create image from array
img = Image.fromarray(rgba)
return img, stats
def band_data_to_image(band_data, colormap):
"""
Creates an python image from pixel values of a GDALRaster.
The input is a dictionary that maps pixel values to RGBA UInt8 colors.
"""
# Get data as 1D array
dat = band_data.ravel()
# Create zeros array
rgba = numpy.zeros((dat.shape[0], 4), dtype='uint8')
# Replace matched rows with colors
stats = {}
for key, color in colormap.items():
orig_key = key
try:
# Try to use the key as number directly
key = float(key)
selector = dat == key
except ValueError:
# Otherwise use it as numpy expression directly
parser = FormulaParser()
selector = parser.evaluate({'x': dat}, key)
# If masked, use mask to filter values additional to formula values
if numpy.ma.is_masked(selector):
selector.fill_value = False
rgba[selector.filled() == 1] = color
# Compress for getting statistics
selector = selector.compressed()
else:
rgba[selector] = color
# Track pixel statistics for this tile
stats[orig_key] = int(numpy.sum(selector))
# Reshape array to image size
rgba = rgba.reshape(band_data.shape[0], band_data.shape[1], 4)
# Create image from array
img = Image.fromarray(rgba)
return img, stats
def value_count(self):
"""
Compute aggregate statistics for a layers dictionary, potentially for
an algebra expression and clipped by a geometry.
The grouping parameter specifies how to group the pixel values for the
aggregation count.
Allowed are the following options:
* 'auto' (the default). The output will be grouped by unique values if all
input rasters are discrete, otherwise a numpy histogram is used.
* 'discrete' groups the data will be grouped by unique values
* 'continuous' groups the data in a numpy histogram
* If an integer value is passed to the argument, it is interpreted as a
legend_id. The data will be grouped using the legend expressions. For
For instance, use grouping=23 for grouping the output with legend 23.
"""
# Instantiate counter dict for value counts.
results = Counter({})
self._clear_stats()
if self.memory_efficient:
# Loop through tiles individually.
all_result_data = self.tiles()
else:
# Combine all tiles into one big array.
all_result_data = [tile for tile in self.tiles()]
if len(all_result_data):
all_result_data = (
numpy.concatenate(all_result_data),
)
for result_data in all_result_data:
if self.grouping == 'discrete':
# Compute unique counts for discrete input data
unique_counts = numpy.unique(result_data, return_counts=True)
# Add counts to results
values = dict(zip(unique_counts[0], unique_counts[1]))
elif self.grouping == 'continuous':
if self.memory_efficient and not self.hist_range:
raise RasterAggregationException(
'Secify a histogram range for memory efficient continuous aggregation.'
)
# Handle continuous case - compute histogram on masked data
counts, bins = numpy.histogram(result_data, range=self.hist_range)
# Create dictionary with bins as keys and histogram counts as values
values = {}
for i in range(len(bins) - 1):
values[(bins[i], bins[i + 1])] = counts[i]
else:
# If input is not a legend, interpret input as legend json data
if not isinstance(self.grouping, Legend):
self.grouping = Legend(json=self.grouping)
# Try getting a colormap from the input
try:
colormap = self.grouping.colormap
except:
raise RasterAggregationException(
'Invalid grouping value found for valuecount.'
)
# Use colormap to compute value counts
formula_parser = FormulaParser()
values = {}
for key, color in colormap.items():
try:
# Try to use the key as number directly
selector = result_data == float(key)
except ValueError:
# Otherwise use it as numpy expression directly
selector = formula_parser.evaluate({'x': result_data}, key)
values[key] = numpy.sum(selector)
# Add counts to results.
results.update(Counter(values))
# Push statistics.
self._push_stats(result_data)
# Transform pixel count to acres if requested
scaling_factor = 1
if self.acres and self.rastgeom and len(results):
scaling_factor = abs(self.rastgeom.scale.x * self.rastgeom.scale.y) * 0.000247105381
results = {
str(int(k) if type(k) == numpy.float64 and int(k) == k else k):
v * scaling_factor for k, v in results.items()
}
return results
def value_count(self):
"""
Compute aggregate statistics for a layers dictionary, potentially for
an algebra expression and clipped by a geometry.
The grouping parameter specifies how to group the pixel values for the
aggregation count.
Allowed are the following options:
* 'auto' (the default). The output will be grouped by unique values if all
input rasters are discrete, otherwise a numpy histogram is used.
* 'discrete' groups the data will be grouped by unique values
* 'continuous' groups the data in a numpy histogram
* If an integer value is passed to the argument, it is interpreted as a
legend_id. The data will be grouped using the legend expressions. For
For instance, use grouping=23 for grouping the output with legend 23.
"""
results = Counter({})
for result_data in self.tiles():
if self.grouping == 'discrete':
# Compute unique counts for discrete input data
unique_counts = numpy.unique(result_data.compressed(),
return_counts=True)
# Add counts to results
values = dict(zip(unique_counts[0], unique_counts[1]))
elif self.grouping == 'continuous':
# Handle continuous case - compute histogram on masked (compressed) data
counts, bins = numpy.histogram(result_data.compressed())
# Create dictionary with bins as keys and histogram counts as values
values = {}
for i in range(len(bins) - 1):
values[(bins[i], bins[i + 1])] = counts[i]
else:
# If input is not a legend, interpret input as legend json data
if not isinstance(self.grouping, Legend):
self.grouping = Legend(json=self.grouping)
# Try getting a colormap from the input
try:
colormap = self.grouping.colormap
except:
raise RasterAggregationException(
'Invalid grouping value found for valuecount.')
# Use colormap to compute value counts
formula_parser = FormulaParser()
values = {}
for key, color in colormap.items():
try:
# Try to use the key as number directly
selector = result_data.compressed() == float(key)
except ValueError:
# Otherwise use it as numpy expression directly
selector = formula_parser.evaluate(
{'x': result_data.compressed()}, key)
values[key] = numpy.sum(selector)
# Add counts to results
results += Counter(values)
# Transform pixel count to acres if requested
scaling_factor = 1
if self.acres and self.rastgeom and len(results):
scaling_factor = abs(
self.rastgeom.scale.x * self.rastgeom.scale.y) * 0.000247105381
results = {
str(int(k) if type(k) == numpy.float64 and int(k) == k else k):
v * scaling_factor
for k, v in results.items()
}
return results
class FormulaParserTests(TestCase):
def setUp(self):
self.parser = FormulaParser()
def assertFormulaResult(self, formula, expVal, data={}):
# Get data from test object if not passed as an argument
if hasattr(self, 'data'):
data = self.data
# Evaluate
val = self.parser.evaluate(data, formula)
# Drop nan values
if data and isinstance(expVal, (list, tuple, numpy.ndarray)) and any(
numpy.array(numpy.isnan(expVal))):
val = val[numpy.logical_not(numpy.isnan(val))]
expVal = expVal[numpy.logical_not(numpy.isnan(expVal))]
# Assert non nan values are as expected
if isinstance(val, numpy.ndarray):
val = val.tolist()
if isinstance(expVal, numpy.ndarray):
expVal = expVal.tolist()
self.assertEqual(val, expVal)
# Check that nan values are the same
self.assertTrue(
numpy.array_equal(numpy.isnan(expVal), numpy.isnan(val)))
def test_formula_parser_without_vars(self):
self.assertFormulaResult("9", 9)
self.assertFormulaResult("-9", -9)
self.assertFormulaResult("--9", 9)
self.assertFormulaResult("+9 - 8", 1)
self.assertFormulaResult("9 + 3 + 6", 9 + 3 + 6)
self.assertFormulaResult("9 + 3 / 11", 9 + 3.0 / 11)
self.assertFormulaResult("(9 + 3)", (9 + 3))
self.assertFormulaResult("(9 + 3) / 11", (9 + 3.0) / 11)
self.assertFormulaResult("9 - 12 - 6", 9 - 12 - 6)
self.assertFormulaResult("9 - (12 - 6)", 9 - (12 - 6))
self.assertFormulaResult("2 * 3.14159", 2 * 3.14159)
self.assertFormulaResult("3.1415926535 * 3.1415926535 / 10",
3.1415926535 * 3.1415926535 / 10)
self.assertFormulaResult("PI * PI / 10", numpy.pi * numpy.pi / 10)
self.assertFormulaResult("PI * PI / 10", numpy.pi * numpy.pi / 10)
self.assertFormulaResult("PI^2", numpy.pi**2)
self.assertFormulaResult("round(PI^2)", round(numpy.pi**2))
self.assertFormulaResult("6.02E23 * 8.048", 6.02E23 * 8.048)
self.assertFormulaResult("6.02e23 * 8.048", 6.02E23 * 8.048)
self.assertFormulaResult("sin(PI / 2)", numpy.sin(numpy.pi / 2))
self.assertFormulaResult("cos(PI / 5)", numpy.cos(numpy.pi / 5))
self.assertFormulaResult("-E", -numpy.e)
self.assertFormulaResult("int(E)", int(numpy.e))
self.assertFormulaResult("int(-E)", int(-numpy.e))
self.assertFormulaResult("round(E)", round(numpy.e))
self.assertFormulaResult("round(-E)", round(-numpy.e))
self.assertFormulaResult("E^PI", numpy.e**numpy.pi)
self.assertFormulaResult("2^1^3^2", 2**1**3**2)
self.assertFormulaResult("2^0", 1)
self.assertFormulaResult("2^3^(2 + 1)", 2**3**3)
self.assertFormulaResult("2^3 + 2", 2**3 + 2)
self.assertFormulaResult("2^9", 2**9)
self.assertFormulaResult("sign(-2)", -1)
self.assertFormulaResult("sign(0)", 0)
self.assertFormulaResult("sign(0.1)", 1)
self.assertFormulaResult("exp(0)", 1)
self.assertFormulaResult("exp(1)", numpy.e)
self.assertFormulaResult("log(1)", 0)
self.assertFormulaResult("TRUE", True)
self.assertFormulaResult("FALSE", False)
self.assertFormulaResult("FALSE | TRUE", True)
self.assertFormulaResult("TRUE & TRUE", True)
self.assertFormulaResult("TRUE & FALSE", False)
self.assertFormulaResult("2 > 1", True)
self.assertFormulaResult("2 >= 1", True)
self.assertFormulaResult("2 < 1", False)
self.assertFormulaResult("2 <= 1", False)
self.assertFormulaResult("1 == 1", True)
self.assertFormulaResult("1 != 2", True)
self.assertFormulaResult("!TRUE", False)
self.assertFormulaResult("!1", False)
self.assertFormulaResult("!-1", False)
self.assertFormulaResult("!2", False)
self.assertFormulaResult("2 * 3 * 4 * 5", 2 * 3 * 4 * 5)
self.assertFormulaResult("2 + 3 + 4 + 5", 2 + 3 + 4 + 5)
self.assertFormulaResult(
"(2 + 3 * 6) * ((2 + 3 * (3 * 4 + 1)) + 4 + 5)", 1000)
self.assertFormulaResult("INF > 0", True)
self.assertFormulaResult("INF < 0", False)
self.assertFormulaResult("-INF > 0", False)
self.assertFormulaResult("~300", 300)
def test_formula_parser_with_vars(self):
d = self.data = {
"a": numpy.array([2, 4, 6]),
"b": numpy.array([True, False, True]),
"c": numpy.array([True, False, False]),
"d": [2, 4, 6],
"x": numpy.array([1.2, 0, -1.2]),
"y": numpy.array([0, 1, 0]),
"z": numpy.array([-5, 78, 912]),
"u": numpy.array([2, 4, 6]),
"A": numpy.array([1E5, 2.3e4]),
"e": numpy.array([2, 11e3]),
"aLongVariable": numpy.array([1, 2, 3]),
"A1A": numpy.array([1, 2, 3]),
"a_1_a": numpy.array([1, 2, 3]),
}
self.assertFormulaResult("-x", -d['x'])
self.assertFormulaResult("sin(x)", numpy.sin(d['x']))
self.assertFormulaResult("cos(x)", numpy.cos(d['x']))
self.assertFormulaResult("tan(x)", numpy.tan(d['x']))
self.assertFormulaResult("log(a)", numpy.log(d['a']))
self.assertFormulaResult("abs(x)", numpy.abs(d['x']))
self.assertFormulaResult("round(x)", numpy.round(d['x']))
self.assertFormulaResult("sign(x)", numpy.sign(d['x']))
self.assertFormulaResult("x == 0", [0, 1, 0])
self.assertFormulaResult("INF < x", [False, False, False])
self.assertFormulaResult("x > 0", [1, 0, 0])
self.assertFormulaResult("x < 0", [0, 0, 1])
self.assertFormulaResult("x >= 1.0", [1, 0, 0])
self.assertFormulaResult("x >= 1.2", [1, 0, 0])
self.assertFormulaResult("x <= 0", [0, 1, 1])
self.assertFormulaResult("x <= 1", [0, 1, 1])
self.assertFormulaResult("(x >= 0) & (x <= 0)", [0, 1, 0])
self.assertFormulaResult("(x > 1) | (x >= 0)", [1, 1, 0])
self.assertFormulaResult("x + x", [2.4, 0, -2.4])
self.assertFormulaResult("x - x", [0, 0, 0])
self.assertFormulaResult("x * x", [1.2 * 1.2, 0, -1.2 * -1.2])
self.assertFormulaResult("x + y", d['x'] + d['y'])
self.assertFormulaResult("x / (x + y)", [1, 0, 1])
self.assertFormulaResult("x * 99999 + 0.5 * y",
[1.2 * 99999, 0.5, -1.2 * 99999])
self.assertFormulaResult("x + y + z + a",
d['x'] + d['y'] + d['z'] + d['a'])
self.assertFormulaResult("a ^ 2", [4, 16, 36])
self.assertFormulaResult("b & c", [True, False, False])
self.assertFormulaResult("b | c", [True, False, True])
self.assertFormulaResult("b == TRUE", [True, False, True])
self.assertFormulaResult("(b == TRUE) & (c == FALSE)",
[False, False, True])
self.assertFormulaResult("(b == TRUE) | (c == FALSE)",
[True, True, True])
self.assertFormulaResult("d", d['d'])
# Mix euler number, scientific notation and variable called e
self.assertFormulaResult("E * 1e5 * e", numpy.e * 1e5 * d['e'])
# Nested expressions can be evaluated
self.assertFormulaResult("((a * 2) + (x * 3)) * 6",
[(2 * 2 + 1.2 * 3) * 6, (4 * 2 + 0 * 3) * 6,
(6 * 2 + -1.2 * 3) * 6])
# Formula is case sensitive
self.assertFormulaResult("-A", -d['A'])
# Long variable name is accepted
self.assertFormulaResult("-aLongVariable", -d['aLongVariable'])
# Alphanumeric variable name
self.assertFormulaResult("-A1A", -d['A1A'])
# Alphanumeric variable name with underscore
self.assertFormulaResult("-a_1_a", -d['a_1_a'])
# Formula with Linebreaks and white space
self.assertFormulaResult("\n x \n + \r y \r + z",
d['x'] + d['y'] + d['z'])
# Long formulas mixing logical with numerical expressions
self.assertFormulaResult('x*(x>1)', d['x'] * (d['x'] > 1))
self.assertFormulaResult('(b>0) & (a > 1) & (c < 1) & (d > 0)',
(d['b'] > 0) & (d['a'] > 1) & (d['c'] < 1) &
(d['a'] > 0))
self.assertFormulaResult(
'(a*b)*((b>0) & (a > 1)) + 99*a*(b <=0)', d['a'] * d['b'] *
((d['b'] > 0) & (d['a'] > 0)) + 99 * d['a'] * (d['b'] <= 0))
self.assertFormulaResult(
'x*(x>1) + 2*y + 3*z*(z==78)',
d['x'] * (d['x'] > 1) + 2 * d['y'] + 3 * d['z'] * (d['z'] == 78),
)
self.assertFormulaResult('a*(a<=1)+(a>1)',
d['a'] * (d['a'] <= 1) + (d['a'] > 1))
self.assertFormulaResult("~a", d['a'])
def test_formula_parser_with_unknown_vars(self):
# Unknown variable raises error
msg = 'Found an undeclared variable "not_a_var" in formula.'
with self.assertRaisesMessage(RasterAlgebraException, msg):
self.parser.evaluate({}, "3 * not_a_var")
def test_formula_parser_without_formula(self):
# Unknown variable raises error
msg = 'Formula not specified.'
with self.assertRaisesMessage(RasterAlgebraException, msg):
self.parser.evaluate({})
def test_with_null_comparison(self):
mask = [True, True, False]
self.data = {
"masked": numpy.ma.masked_array([1, 2, 3], mask=mask),
"unmasked": numpy.array([1, 2, 3]),
}
self.assertFormulaResult('masked == NULL', mask)
self.assertFormulaResult('masked != NULL', [not x for x in mask])
self.assertFormulaResult('NULL == masked', mask)
self.assertFormulaResult('unmasked == NULL', [False, False, False])
msg = 'NULL can only be used with "==" or "!=" operators.'
with self.assertRaisesMessage(RasterAlgebraException, msg):
self.assertFormulaResult('masked >= NULL', mask)
# Null value propagation in formula
self.assertFormulaResult(
"unmasked * (masked != NULL) + 99 * (masked == NULL)", [99, 99, 3])
# Fill operator removes null values
self.assertFormulaResult("~masked == NULL", [False, False, False])
def test_masked_array_result(self):
data = {
'x':
numpy.ma.masked_array([1, 2, 3],
mask=[True, False, False],
fill_value=99),
'y':
numpy.ma.masked_array([1, 2, 3],
mask=[False, True, False],
fill_value=100),
}
result = self.parser.evaluate(data, 'x + y')
self.assertTrue(numpy.ma.is_masked(result))
self.assertEqual(result.compressed().tolist(),
(data['x'] + data['y']).compressed().tolist())
def test_fill_operator_result(self):
data = {
'x':
numpy.ma.masked_array([1, 2, 3],
mask=[True, False, False],
fill_value=99),
'y':
numpy.ma.masked_array([1, 2, 3],
mask=[False, True, False],
fill_value=100),
}
result = self.parser.evaluate(data, '~x + y')
self.assertEqual(result.compressed().tolist(),
(data['x'].filled() +
data['y']).compressed().tolist())
result = self.parser.evaluate(data, '~x + ~y')
self.assertEqual(result.tolist(),
(data['x'].filled() + data['y'].filled()).tolist())
result = self.parser.evaluate(data, '~x + (~y == NULL)')
self.assertEqual(result.tolist(), (data['x'].filled()).tolist())
def test_re_evaluation(self):
self.parser.set_formula('+x')
self.assertEqual(self.parser.evaluate({'x': 1}), 1)
self.assertEqual(self.parser.evaluate({'x': 2}), 2)
self.parser.set_formula('-x')
self.assertEqual(self.parser.evaluate({'x': 3}), -3)
self.assertEqual(self.parser.evaluate({'x': 4}), -4)
def test_statistics_functions(self):
d = self.data = {'x': numpy.random.rand(10), 'y': range(3)}
self.assertFormulaResult('min(x)', numpy.min(d['x']))
self.assertFormulaResult('max(x)', numpy.max(d['x']))
self.assertFormulaResult('mean(x)', numpy.mean(d['x']))
self.assertFormulaResult('median(x)', numpy.median(d['x']))
self.assertFormulaResult('std(x)', numpy.std(d['x']))
self.assertFormulaResult('y * min(x)',
numpy.arange(3) * numpy.min(d['x']))
def test_keyword_variable_exception(self):
msg = 'Invalid variable name found: "del".'
with self.assertRaisesMessage(RasterAlgebraException, msg):
self.parser.evaluate({'del': [1]}, 'del')
def test_invalid_variables(self):
d = self.data = {
'_no': range(5),
'no_': range(5),
'__no__': range(5),
'__': range(5),
}
for key, val in d.items():
msg = 'Invalid variable name found: "{}".'.format(key)
with self.assertRaisesMessage(RasterAlgebraException, msg):
self.parser.evaluate({key: val}, key)
def setUp(self):
self.parser = FormulaParser()
class FormulaParserTests(TestCase):
def setUp(self):
self.parser = FormulaParser()
def assertFormulaResult(self, formula, expVal, data={}):
# Get data from test object if not passed as an argument
if hasattr(self, 'data'):
data = self.data
# Evaluate
val = self.parser.evaluate(data, formula)
# Drop nan values
if data and isinstance(expVal, (list, tuple, numpy.ndarray)) and any(numpy.array(numpy.isnan(expVal))):
val = val[numpy.logical_not(numpy.isnan(val))]
expVal = expVal[numpy.logical_not(numpy.isnan(expVal))]
# Assert non nan values are as expected
if isinstance(val, numpy.ndarray):
val = val.tolist()
if isinstance(expVal, numpy.ndarray):
expVal = expVal.tolist()
self.assertEqual(val, expVal)
# Check that nan values are the same
self.assertTrue(numpy.array_equal(numpy.isnan(expVal), numpy.isnan(val)))
def test_formula_parser_without_vars(self):
self.assertFormulaResult("9", 9)
self.assertFormulaResult("-9", -9)
self.assertFormulaResult("--9", 9)
self.assertFormulaResult("+9 - 8", 1)
self.assertFormulaResult("9 + 3 + 6", 9 + 3 + 6)
self.assertFormulaResult("9 + 3 / 11", 9 + 3.0 / 11)
self.assertFormulaResult("(9 + 3)", (9 + 3))
self.assertFormulaResult("(9 + 3) / 11", (9 + 3.0) / 11)
self.assertFormulaResult("9 - 12 - 6", 9 - 12 - 6)
self.assertFormulaResult("9 - (12 - 6)", 9 - (12 - 6))
self.assertFormulaResult("2 * 3.14159", 2 * 3.14159)
self.assertFormulaResult("3.1415926535 * 3.1415926535 / 10", 3.1415926535 * 3.1415926535 / 10)
self.assertFormulaResult("PI * PI / 10", numpy.pi * numpy.pi / 10)
self.assertFormulaResult("PI * PI / 10", numpy.pi * numpy.pi / 10)
self.assertFormulaResult("PI^2", numpy.pi ** 2)
self.assertFormulaResult("round(PI^2)", round(numpy.pi ** 2))
self.assertFormulaResult("6.02E23 * 8.048", 6.02E23 * 8.048)
self.assertFormulaResult("6.02e23 * 8.048", 6.02E23 * 8.048)
self.assertFormulaResult("sin(PI / 2)", numpy.sin(numpy.pi / 2))
self.assertFormulaResult("cos(PI / 5)", numpy.cos(numpy.pi / 5))
self.assertFormulaResult("-E", -numpy.e)
self.assertFormulaResult("int(E)", int(numpy.e))
self.assertFormulaResult("int(-E)", int(-numpy.e))
self.assertFormulaResult("round(E)", round(numpy.e))
self.assertFormulaResult("round(-E)", round(-numpy.e))
self.assertFormulaResult("E^PI", numpy.e ** numpy.pi)
self.assertFormulaResult("2^1^3^2", 2 ** 1 ** 3 ** 2)
self.assertFormulaResult("2^0", 1)
self.assertFormulaResult("2^3^(2 + 1)", 2 ** 3 ** 3)
self.assertFormulaResult("2^3 + 2", 2 ** 3 + 2)
self.assertFormulaResult("2^9", 2 ** 9)
self.assertFormulaResult("sign(-2)", -1)
self.assertFormulaResult("sign(0)", 0)
self.assertFormulaResult("sign(0.1)", 1)
self.assertFormulaResult("exp(0)", 1)
self.assertFormulaResult("exp(1)", numpy.e)
self.assertFormulaResult("log(1)", 0)
self.assertFormulaResult("TRUE", True)
self.assertFormulaResult("FALSE", False)
self.assertFormulaResult("FALSE | TRUE", True)
self.assertFormulaResult("TRUE & TRUE", True)
self.assertFormulaResult("TRUE & FALSE", False)
self.assertFormulaResult("2 > 1", True)
self.assertFormulaResult("2 >= 1", True)
self.assertFormulaResult("2 < 1", False)
self.assertFormulaResult("2 <= 1", False)
self.assertFormulaResult("1 == 1", True)
self.assertFormulaResult("1 != 2", True)
self.assertFormulaResult("!TRUE", False)
self.assertFormulaResult("!1", False)
self.assertFormulaResult("!-1", False)
self.assertFormulaResult("!2", False)
self.assertFormulaResult("2 * 3 * 4 * 5", 2 * 3 * 4 * 5)
self.assertFormulaResult("2 + 3 + 4 + 5", 2 + 3 + 4 + 5)
self.assertFormulaResult("(2 + 3 * 6) * ((2 + 3 * (3 * 4 + 1)) + 4 + 5)", 1000)
self.assertFormulaResult("INF > 0", True)
self.assertFormulaResult("INF < 0", False)
self.assertFormulaResult("-INF > 0", False)
self.assertFormulaResult("~300", 300)
def test_formula_parser_with_vars(self):
d = self.data = {
"a": numpy.array([2, 4, 6]),
"b": numpy.array([True, False, True]),
"c": numpy.array([True, False, False]),
"d": [2, 4, 6],
"x": numpy.array([1.2, 0, -1.2]),
"y": numpy.array([0, 1, 0]),
"z": numpy.array([-5, 78, 912]),
"u": numpy.array([2, 4, 6]),
"A": numpy.array([1E5, 2.3e4]),
"e": numpy.array([2, 11e3]),
"aLongVariable": numpy.array([1, 2, 3]),
"A1A": numpy.array([1, 2, 3]),
"a_1_a": numpy.array([1, 2, 3]),
}
self.assertFormulaResult("-x", -d['x'])
self.assertFormulaResult("sin(x)", numpy.sin(d['x']))
self.assertFormulaResult("cos(x)", numpy.cos(d['x']))
self.assertFormulaResult("tan(x)", numpy.tan(d['x']))
self.assertFormulaResult("log(a)", numpy.log(d['a']))
self.assertFormulaResult("abs(x)", numpy.abs(d['x']))
self.assertFormulaResult("round(x)", numpy.round(d['x']))
self.assertFormulaResult("sign(x)", numpy.sign(d['x']))
self.assertFormulaResult("x == 0", [0, 1, 0])
self.assertFormulaResult("INF < x", [False, False, False])
self.assertFormulaResult("x > 0", [1, 0, 0])
self.assertFormulaResult("x < 0", [0, 0, 1])
self.assertFormulaResult("x >= 1.0", [1, 0, 0])
self.assertFormulaResult("x >= 1.2", [1, 0, 0])
self.assertFormulaResult("x <= 0", [0, 1, 1])
self.assertFormulaResult("x <= 1", [0, 1, 1])
self.assertFormulaResult("(x >= 0) & (x <= 0)", [0, 1, 0])
self.assertFormulaResult("(x > 1) | (x >= 0)", [1, 1, 0])
self.assertFormulaResult("x + x", [2.4, 0, -2.4])
self.assertFormulaResult("x - x", [0, 0, 0])
self.assertFormulaResult("x * x", [1.2 * 1.2, 0, -1.2 * -1.2])
self.assertFormulaResult("x + y", d['x'] + d['y'])
self.assertFormulaResult("x / (x + y)", [1, 0, 1])
self.assertFormulaResult("x * 99999 + 0.5 * y", [1.2 * 99999, 0.5, -1.2 * 99999])
self.assertFormulaResult("x + y + z + a", d['x'] + d['y'] + d['z'] + d['a'])
self.assertFormulaResult("a ^ 2", [4, 16, 36])
self.assertFormulaResult("b & c", [True, False, False])
self.assertFormulaResult("b | c", [True, False, True])
self.assertFormulaResult("b == TRUE", [True, False, True])
self.assertFormulaResult("(b == TRUE) & (c == FALSE)", [False, False, True])
self.assertFormulaResult("(b == TRUE) | (c == FALSE)", [True, True, True])
self.assertFormulaResult("d", d['d'])
# Mix euler number, scientific notation and variable called e
self.assertFormulaResult("E * 1e5 * e", numpy.e * 1e5 * d['e'])
# Nested expressions can be evaluated
self.assertFormulaResult(
"((a * 2) + (x * 3)) * 6",
[(2 * 2 + 1.2 * 3) * 6, (4 * 2 + 0 * 3) * 6, (6 * 2 + -1.2 * 3) * 6]
)
# Formula is case sensitive
self.assertFormulaResult("-A", -d['A'])
# Long variable name is accepted
self.assertFormulaResult("-aLongVariable", -d['aLongVariable'])
# Alphanumeric variable name
self.assertFormulaResult("-A1A", -d['A1A'])
# Alphanumeric variable name with underscore
self.assertFormulaResult("-a_1_a", -d['a_1_a'])
# Formula with Linebreaks and white space
self.assertFormulaResult("\n x \n + \r y \r + z", d['x'] + d['y'] + d['z'])
# Long formulas mixing logical with numerical expressions
self.assertFormulaResult('x*(x>1)', d['x'] * (d['x'] > 1))
self.assertFormulaResult(
'(b>0) & (a > 1) & (c < 1) & (d > 0)',
(d['b'] > 0) & (d['a'] > 1) & (d['c'] < 1) & (d['a'] > 0)
)
self.assertFormulaResult(
'(a*b)*((b>0) & (a > 1)) + 99*a*(b <=0)',
d['a'] * d['b'] * ((d['b'] > 0) & (d['a'] > 0)) + 99 * d['a'] * (d['b'] <= 0)
)
self.assertFormulaResult(
'x*(x>1) + 2*y + 3*z*(z==78)',
d['x'] * (d['x'] > 1) + 2 * d['y'] + 3 * d['z'] * (d['z'] == 78),
)
self.assertFormulaResult('a*(a<=1)+(a>1)', d['a'] * (d['a'] <= 1) + (d['a'] > 1))
self.assertFormulaResult("~a", d['a'])
def test_formula_parser_with_unknown_vars(self):
# Unknown variable raises error
msg = 'Found an undeclared variable "not_a_var" in formula.'
with self.assertRaisesMessage(RasterAlgebraException, msg):
self.parser.evaluate({}, "3 * not_a_var")
def test_formula_parser_without_formula(self):
# Unknown variable raises error
msg = 'Formula not specified.'
with self.assertRaisesMessage(RasterAlgebraException, msg):
self.parser.evaluate({})
def test_with_null_comparison(self):
mask = [True, True, False]
self.data = {
"masked": numpy.ma.masked_array([1, 2, 3], mask=mask),
"unmasked": numpy.array([1, 2, 3]),
}
self.assertFormulaResult('masked == NULL', mask)
self.assertFormulaResult('masked != NULL', [not x for x in mask])
self.assertFormulaResult('NULL == masked', mask)
self.assertFormulaResult('unmasked == NULL', [False, False, False])
msg = 'NULL can only be used with "==" or "!=" operators.'
with self.assertRaisesMessage(RasterAlgebraException, msg):
self.assertFormulaResult('masked >= NULL', mask)
# Null value propagation in formula
self.assertFormulaResult("unmasked * (masked != NULL) + 99 * (masked == NULL)", [99, 99, 3])
# Fill operator removes null values
self.assertFormulaResult("~masked == NULL", [False, False, False])
def test_masked_array_result(self):
data = {
'x': numpy.ma.masked_array([1, 2, 3], mask=[True, False, False], fill_value=99),
'y': numpy.ma.masked_array([1, 2, 3], mask=[False, True, False], fill_value=100),
}
result = self.parser.evaluate(data, 'x + y')
self.assertTrue(numpy.ma.is_masked(result))
self.assertEqual(
result.compressed().tolist(),
(data['x'] + data['y']).compressed().tolist()
)
def test_fill_operator_result(self):
data = {
'x': numpy.ma.masked_array([1, 2, 3], mask=[True, False, False], fill_value=99),
'y': numpy.ma.masked_array([1, 2, 3], mask=[False, True, False], fill_value=100),
}
result = self.parser.evaluate(data, '~x + y')
self.assertEqual(
result.compressed().tolist(),
(data['x'].filled() + data['y']).compressed().tolist()
)
result = self.parser.evaluate(data, '~x + ~y')
self.assertEqual(
result.tolist(),
(data['x'].filled() + data['y'].filled()).tolist()
)
result = self.parser.evaluate(data, '~x + (~y == NULL)')
self.assertEqual(
result.tolist(),
(data['x'].filled()).tolist()
)
def test_re_evaluation(self):
self.parser.set_formula('+x')
self.assertEqual(self.parser.evaluate({'x': 1}), 1)
self.assertEqual(self.parser.evaluate({'x': 2}), 2)
self.parser.set_formula('-x')
self.assertEqual(self.parser.evaluate({'x': 3}), -3)
self.assertEqual(self.parser.evaluate({'x': 4}), -4)
def test_statistics_functions(self):
d = self.data = {'x': numpy.random.rand(10), 'y': range(3)}
self.assertFormulaResult('min(x)', numpy.min(d['x']))
self.assertFormulaResult('max(x)', numpy.max(d['x']))
self.assertFormulaResult('mean(x)', numpy.mean(d['x']))
self.assertFormulaResult('median(x)', numpy.median(d['x']))
self.assertFormulaResult('std(x)', numpy.std(d['x']))
self.assertFormulaResult('y * min(x)', numpy.arange(3) * numpy.min(d['x']))
def test_keyword_variable_exception(self):
msg = 'Invalid variable name found: "del".'
with self.assertRaisesMessage(RasterAlgebraException, msg):
self.parser.evaluate({'del': [1]}, 'del')
def test_invalid_variables(self):
d = self.data = {
'_no': range(5),
'no_': range(5),
'__no__': range(5),
'__': range(5),
}
for key, val in d.items():
msg = 'Invalid variable name found: "{}".'.format(key)
with self.assertRaisesMessage(RasterAlgebraException, msg):
self.parser.evaluate({key: val}, key)
def setUp(self):
self.parser = FormulaParser()
def band_data_to_image(band_data, colormap):
"""
Creates an python image from pixel values of a GDALRaster.
The input is a dictionary that maps pixel values to RGBA UInt8 colors.
If an interpolation interval is given, the values are
"""
# Get data as 1D array
dat = band_data.ravel()
stats = {}
if 'continuous' in colormap:
dmin, dmax = colormap.get('range', (dat.min(), dat.max()))
if dmax == dmin:
norm = dat == dmin
else:
norm = (dat - dmin) / (dmax - dmin)
color_from = colormap.get('from', [0, 0, 0])
color_to = colormap.get('to', [1, 1, 1])
color_over = colormap.get('over', [None, None, None])
red = rescale_to_channel_range(norm.copy(), color_from[0], color_to[0],
color_over[0])
green = rescale_to_channel_range(norm.copy(), color_from[1],
color_to[1], color_over[1])
blue = rescale_to_channel_range(norm.copy(), color_from[2],
color_to[2], color_over[2])
# Compute alpha channel from mask if available.
if numpy.ma.is_masked(dat):
alpha = 255 * numpy.logical_not(
dat.mask) * (norm >= 0) * (norm <= 1)
else:
alpha = 255 * (norm > 0) * (norm < 1)
rgba = numpy.array([red, green, blue, alpha], dtype='uint8').T
else:
# Create zeros array
rgba = numpy.zeros((dat.shape[0], 4), dtype='uint8')
# Replace matched rows with colors
for key, color in colormap.items():
orig_key = key
try:
# Try to use the key as number directly
key = float(key)
selector = dat == key
except ValueError:
# Otherwise use it as numpy expression directly
parser = FormulaParser()
selector = parser.evaluate({'x': dat}, key)
# If masked, use mask to filter values additional to formula values
if numpy.ma.is_masked(selector):
selector.fill_value = False
rgba[selector.filled() == 1] = color
# Compress for getting statistics
selector = selector.compressed()
else:
rgba[selector] = color
# Track pixel statistics for this tile
stats[orig_key] = int(numpy.sum(selector))
# Reshape array to image size
rgba = rgba.reshape(band_data.shape[0], band_data.shape[1], 4)
# Create image from array
img = Image.fromarray(rgba)
return img, stats
def band_data_to_image(band_data, colormap):
"""
Creates an python image from pixel values of a GDALRaster.
The input is a dictionary that maps pixel values to RGBA UInt8 colors.
If an interpolation interval is given, the values are
"""
# Get data as 1D array
dat = band_data.ravel()
stats = {}
if 'continuous' in colormap:
dmin, dmax = colormap.get('range', (dat.min(), dat.max()))
if dmax == dmin:
norm = dat == dmin
else:
norm = (dat - dmin) / (dmax - dmin)
color_from = colormap.get('from', [0, 0, 0])
color_to = colormap.get('to', [1, 1, 1])
color_over = colormap.get('over', [None, None, None])
red = rescale_to_channel_range(norm.copy(), color_from[0], color_to[0], color_over[0])
green = rescale_to_channel_range(norm.copy(), color_from[1], color_to[1], color_over[1])
blue = rescale_to_channel_range(norm.copy(), color_from[2], color_to[2], color_over[2])
# Compute alpha channel from mask if available.
if numpy.ma.is_masked(dat):
alpha = 255 * numpy.logical_not(dat.mask) * (norm >= 0) * (norm <= 1)
else:
alpha = 255 * (norm > 0) * (norm < 1)
rgba = numpy.array([red, green, blue, alpha], dtype='uint8').T
else:
# Create zeros array
rgba = numpy.zeros((dat.shape[0], 4), dtype='uint8')
# Replace matched rows with colors
for key, color in colormap.items():
orig_key = key
try:
# Try to use the key as number directly
key = float(key)
selector = dat == key
except ValueError:
# Otherwise use it as numpy expression directly
parser = FormulaParser()
selector = parser.evaluate({'x': dat}, key)
# If masked, use mask to filter values additional to formula values
if numpy.ma.is_masked(selector):
selector.fill_value = False
rgba[selector.filled() == 1] = color
# Compress for getting statistics
selector = selector.compressed()
else:
rgba[selector] = color
# Track pixel statistics for this tile
stats[orig_key] = int(numpy.sum(selector))
# Reshape array to image size
rgba = rgba.reshape(band_data.shape[0], band_data.shape[1], 4)
# Create image from array
img = Image.fromarray(rgba)
return img, stats