def test_reduce_by_expression(self):
task = ReduceByExpression()
arr_1 = numpy.arange(10)
arr_2 = numpy.arange(10, 20)
arr_3 = numpy.arange(20, 30)
expected = arr_1 + arr_2 + arr_3
expr = 'x + y'
result = task(arrays_in=[arr_1, arr_2, arr_3], expression=expr)
array_out = result['array_out']
assert is_ndarray(array_out)
assert (array_out == expected).all()
initial = numpy.arange(10)
expected += initial
result = task(arrays_in=[arr_1, arr_2, arr_3],
expression=expr,
initial_array=initial)
array_out = result['array_out']
assert is_ndarray(array_out)
assert (array_out == expected).all()
result = task(arrays_in=[arr_1, arr_2, arr_3],
expression='x + (y-z)',
z=2)
assert (result['array_out'] == (arr_1 + (arr_2 - 2) +
(arr_3 - 2))).all()
def fillnodata(
image,
mask=None,
max_search_distance=100.0,
smoothing_iterations=0):
"""Fill holes in raster data by interpolation
This algorithm will interpolate values for all designated nodata
pixels (marked by zeros in `mask`). For each pixel a four direction
conic search is done to find values to interpolate from (using
inverse distance weighting). Once all values are interpolated, zero
or more smoothing iterations (3x3 average filters on interpolated
pixels) are applied to smooth out artifacts.
This algorithm is generally suitable for interpolating missing
regions of fairly continuously varying rasters (such as elevation
models for instance). It is also suitable for filling small holes
and cracks in more irregularly varying images (like aerial photos).
It is generally not so great for interpolating a raster from sparse
point data.
Parameters
----------
image : numpy ndarray
The source image with holes to be filled. If a MaskedArray, the
inverse of its mask will define the pixels to be filled --
unless the ``mask`` argument is not None (see below).`
mask : numpy ndarray or None
A mask band indicating which pixels to interpolate. Pixels to
interpolate into are indicated by the value 0. Values
> 0 indicate areas to use during interpolation. Must be same
shape as image. This array always takes precedence over the
image's mask (see above). If None, the inverse of the image's
mask will be used if available.
max_search_distance : float, optional
The maxmimum number of pixels to search in all directions to
find values to interpolate from. The default is 100.
smoothing_iterations : integer, optional
The number of 3x3 smoothing filter passes to run. The default is
0.
Returns
-------
out : numpy ndarray
The filled raster array.
"""
if mask is None and isinstance(image, MaskedArray):
mask = ~image.mask
if not dtypes.is_ndarray(mask):
raise ValueError("An mask array is required")
if isinstance(image, MaskedArray):
image = image.data
if not dtypes.is_ndarray(image):
raise ValueError("An image array is required")
max_search_distance = float(max_search_distance)
smoothing_iterations = int(smoothing_iterations)
return _fillnodata(
image, mask, max_search_distance, smoothing_iterations)
def evaluate_expression(self, expression, context={}):
try:
# Operations against masked arrays are really slow, so take a regular array view, then back to a masked
# array afterwards. Todo: find a better solution long-term
expr_context = {k: v.view(numpy.ndarray) if is_masked(v) else v for k, v in six.iteritems(context)}
result = Parser().evaluate(expression, context=expr_context)
if is_ndarray(result):
for value in six.itervalues(context):
if is_masked(value):
if is_masked(result) and is_masked(value):
result.mask = result.mask | value.mask
elif is_masked(value):
result = numpy.ma.masked_array(result, mask=value.mask)
result = Raster(result, value.extent, value.x_dim, value.y_dim, value.y_increasing)
break
return result
except (SyntaxError, NameError) as e:
raise ExecutionError(
'The expression is invalid ({0}): {1}\nContext: {2}'.format(str(e), expression, str(context)),
self
)
def evaluate_expression(self, expression, context={}):
try:
# Operations against masked arrays are really slow, so take a regular array view, then back to a masked
# array afterwards. Todo: find a better solution long-term
expr_context = {
k: v.view(ndarray) if is_masked(v) else v
for k, v in context.items()
}
result = Parser().evaluate(expression, context=expr_context)
if is_ndarray(result):
for value in context.values():
if is_masked(value):
if is_masked(result) and is_masked(value):
result.mask = result.mask | value.mask
elif is_masked(value):
result = masked_array(result, mask=value.mask)
result = Raster(result, value.extent, value.x_dim,
value.y_dim, value.y_increasing)
break
return result
except (SyntaxError, NameError) as e:
raise ExecutionError(
'The expression is invalid ({0}): {1}\nContext: {2}'.format(
str(e), expression, str(context)), self)
def test_array_from_dataset(self):
task = ArrayFromDataset()
dataset = Dataset(os.path.join(TEST_DATA_DIR, "simple_grid_2d.nc"))
result = task(dataset=dataset, variable="value")
arr = result["array_out"]
assert is_ndarray(arr)
assert arr.shape == (10, 10)
assert (arr == numpy.reshape(numpy.arange(100), (10, 10))).all()
def test_array_from_dataset(self):
task = ArrayFromDataset()
dataset = Dataset(os.path.join(TEST_DATA_DIR, 'simple_grid_2d.nc'))
result = task(dataset=dataset, variable='value')
arr = result['array_out']
assert is_ndarray(arr)
assert arr.shape == (10, 10)
assert (arr == numpy.reshape(numpy.arange(100), (10, 10))).all()
def test_raster(self):
arr = numpy.reshape(numpy.arange(100), (10, 10))
extent = BBox((0, 0, 10, 10))
raster = Raster(arr, extent, 1, 0)
assert isinstance(raster, Raster)
raster = raster[:]
assert isinstance(raster, Raster)
assert isinstance(raster.extent, BBox)
clipped = raster[3:, 3:-1]
assert isinstance(clipped, Raster)
assert isinstance(raster.extent, BBox)
assert clipped.extent.as_list() == [3, 0, 10, 7]
raster.y_increasing = True
clipped = raster[3:, 3:-1]
assert isinstance(clipped, Raster)
assert isinstance(raster.extent, BBox)
assert clipped.extent.as_list() == [3, 3, 10, 10]
clipped = raster[0]
assert not isinstance(clipped, Raster)
assert isinstance(raster.extent, BBox)
assert is_ndarray(clipped)
assert (clipped == numpy.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])).all()
# Make sure binops don't lose extent info
assert isinstance((raster == raster).extent, BBox)
assert isinstance((raster != raster).extent, BBox)
assert isinstance((raster > 2).extent, BBox)
assert isinstance((2 > raster).extent, BBox)
assert isinstance((raster + 2).extent, BBox)
assert isinstance((2 + raster).extent, BBox)
assert isinstance((raster - 2).extent, BBox)
assert isinstance((2 - raster).extent, BBox)
assert isinstance((raster * 2).extent, BBox)
assert isinstance((2 * raster).extent, BBox)
assert isinstance((raster / 2).extent, BBox)
assert isinstance((2 / raster).extent, BBox)
assert isinstance((raster // 2).extent, BBox)
assert isinstance((2 // raster).extent, BBox)
assert isinstance((raster ** 2).extent, BBox)
assert isinstance((2 ** raster).extent, BBox)
assert isinstance((raster % 2).extent, BBox)
assert isinstance((2 % raster).extent, BBox)
assert isinstance((raster << 2).extent, BBox)
assert isinstance((2 << raster).extent, BBox)
assert isinstance((raster >> 2).extent, BBox)
assert isinstance((2 >> raster).extent, BBox)
assert isinstance((raster | 2).extent, BBox)
assert isinstance((2 | raster).extent, BBox)
assert isinstance((raster & 2).extent, BBox)
assert isinstance((2 & raster).extent, BBox)
assert isinstance((raster ^ 2).extent, BBox)
assert isinstance((2 ^ raster).extent, BBox)
def test_raster(self):
arr = numpy.reshape(numpy.arange(100), (10, 10))
extent = BBox((0, 0, 10, 10))
raster = Raster(arr, extent, 1, 0)
assert isinstance(raster, Raster)
raster = raster[:]
assert isinstance(raster, Raster)
assert isinstance(raster.extent, BBox)
clipped = raster[3:, 3:-1]
assert isinstance(clipped, Raster)
assert isinstance(raster.extent, BBox)
assert clipped.extent.as_list() == [3, 0, 10, 7]
raster.y_increasing = True
clipped = raster[3:, 3:-1]
assert isinstance(clipped, Raster)
assert isinstance(raster.extent, BBox)
assert clipped.extent.as_list() == [3, 3, 10, 10]
clipped = raster[0]
assert not isinstance(clipped, Raster)
assert isinstance(raster.extent, BBox)
assert is_ndarray(clipped)
assert (clipped == numpy.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])).all()
# Make sure binops don't lose extent info
assert isinstance((raster == raster).extent, BBox)
assert isinstance((raster != raster).extent, BBox)
assert isinstance((raster > 2).extent, BBox)
assert isinstance((2 > raster).extent, BBox)
assert isinstance((raster + 2).extent, BBox)
assert isinstance((2 + raster).extent, BBox)
assert isinstance((raster - 2).extent, BBox)
assert isinstance((2 - raster).extent, BBox)
assert isinstance((raster * 2).extent, BBox)
assert isinstance((2 * raster).extent, BBox)
assert isinstance((raster / 2).extent, BBox)
assert isinstance((2 / raster).extent, BBox)
assert isinstance((raster // 2).extent, BBox)
assert isinstance((2 // raster).extent, BBox)
assert isinstance((raster**2).extent, BBox)
assert isinstance((2**raster).extent, BBox)
assert isinstance((raster % 2).extent, BBox)
assert isinstance((2 % raster).extent, BBox)
assert isinstance((raster << 2).extent, BBox)
assert isinstance((2 << raster).extent, BBox)
assert isinstance((raster >> 2).extent, BBox)
assert isinstance((2 >> raster).extent, BBox)
assert isinstance((raster | 2).extent, BBox)
assert isinstance((2 | raster).extent, BBox)
assert isinstance((raster & 2).extent, BBox)
assert isinstance((2 & raster).extent, BBox)
assert isinstance((raster ^ 2).extent, BBox)
assert isinstance((2 ^ raster).extent, BBox)
def _to_ndarray(self, a):
"""Casts Python lists and tuples to a numpy array or raises an AssertionError."""
if isinstance(a, (list, tuple)):
a = numpy.array(a)
if not is_ndarray(a):
raise TypeError("Expected an ndarray but got object of type '{}' instead".format(type(a)))
return a
def _to_ndarray(self, a):
"""Casts Python lists and tuples to a numpy array or raises an AssertionError."""
if isinstance(a, (list, tuple)):
a = numpy.array(a)
if not is_ndarray(a):
raise TypeError("Expected an ndarray but got object of type '{}' instead".format(type(a)))
return a
def test_apply_expression(self):
task = ApplyExpression()
arr = numpy.array([1, 2, 3])
expr = "(x ** 2) / 2"
result = task(array_in=arr, expression=expr)
assert is_ndarray(result["array_out"])
assert (result["array_out"] == numpy.array([0.5, 2, 4.5])).all()
result = task(array_in=arr, expression="(x ** y) / y", y=2)
assert (result["array_out"] == numpy.array([0.5, 2, 4.5])).all()
def test_apply_expression(self):
task = ApplyExpression()
arr = numpy.array([1, 2, 3])
expr = '(x ** 2) / 2'
result = task(array_in=arr, expression=expr)
assert is_ndarray(result['array_out'])
assert (result['array_out'] == numpy.array([.5, 2, 4.5])).all()
result = task(array_in=arr, expression='(x ** y) / y', y=2)
assert (result['array_out'] == numpy.array([.5, 2, 4.5])).all()
def fn_abs(self, value):
"""
Return the absolute value of a number.
:param value: The number.
:return: The absolute value of the number.
"""
if is_ndarray(value):
return numpy.absolute(value)
else:
return abs(value)
def fn_int32(self, value):
"""
Return the value cast to an 64-bit signed integer (numpy array) or a Python int (single value)
:param value: The number or array
:return: The number or array as int/int8
"""
if is_ndarray(value) or isinstance(value, (list, tuple)):
return self._to_ndarray(value).astype(numpy.int64)
else:
return int(value)
def fn_ceil(self, value):
"""
Return the ceiling of a number.
:param value: The number.
:return: The ceiling of the number.
"""
if is_ndarray(value) or isinstance(value, (list, tuple)):
return numpy.ceil(self._to_ndarray(value))
else:
return math.ceil(value)
def fn_floor(self, value):
"""
Return the floor of a number. For negative numbers, floor returns a lower value. E.g., `floor(-2.5) == -3`
:param value: The number.
:return: The floor of the number.
"""
if is_ndarray(value) or isinstance(value, (list, tuple)):
return numpy.floor(self._to_ndarray(value))
else:
return math.floor(value)
def fn_float64(self, value):
"""
Return the value cast to a 64-bit float (numpy array) or a Python float (single value).
:param value: The number.
:return: The number as a float.
"""
if is_ndarray(value) or isinstance(value, (list, tuple)):
return self._to_ndarray(value).astype('float64')
else:
return float(value)
def fn_int64(self, value):
"""
Return the value cast to an 64-bit signed integer (numpy array) or a Python int (single value)
:param value: The number or array
:return: The number or array as int/int8
"""
if is_ndarray(value) or isinstance(value, (list, tuple)):
return self._to_ndarray(value).astype(numpy.int64)
else:
return int(value)
def fn_ceil(self, value):
"""
Return the ceiling of a number.
:param value: The number.
:return: The ceiling of the number.
"""
if is_ndarray(value) or isinstance(value, (list, tuple)):
return numpy.ceil(self._to_ndarray(value))
else:
return math.ceil(value)
def fn_float64(self, value):
"""
Return the value cast to a 64-bit float (numpy array) or a Python float (single value).
:param value: The number.
:return: The number as a float.
"""
if is_ndarray(value) or isinstance(value, (list, tuple)):
return self._to_ndarray(value).astype('float64')
else:
return float(value)
def fn_float(self, value):
"""
Return the value cast to a float.
:param value: The number.
:return: The number as a float.
"""
if is_ndarray(value) or isinstance(value, (list, tuple)):
return self._to_ndarray(value).astype('float')
else:
return float(value)
def fn_float(self, value):
"""
Return the value cast to a float.
:param value: The number.
:return: The number as a float.
"""
if is_ndarray(value) or isinstance(value, (list, tuple)):
return self._to_ndarray(value).astype('float')
else:
return float(value)
def fn_floor(self, value):
"""
Return the floor of a number. For negative numbers, floor returns a lower value. E.g., `floor(-2.5) == -3`
:param value: The number.
:return: The floor of the number.
"""
if is_ndarray(value) or isinstance(value, (list, tuple)):
return numpy.floor(self._to_ndarray(value))
else:
return math.floor(value)
def fn_abs(self, value):
"""
Return the absolute value of a number.
:param value: The number.
:return: The absolute value of the number.
"""
if is_ndarray(value):
return numpy.absolute(value)
else:
return abs(value)
def resolve_item(obj, index):
if is_ndarray(obj) or isinstance(obj, (list, tuple)):
if not isinstance(index, int):
raise TypeError("Not a valid array index: '{}'".format(index))
elif isinstance(obj, dict):
if not isinstance(index, (str, int)):
raise TypeError("Not a valid dictionary index: '{}'".format(index))
else:
raise TypeError("Object does not support indexing: '{}'".format(type(obj)))
return obj[index]
def resolve_item(obj, index):
if is_ndarray(obj) or isinstance(obj, (list, tuple)):
if not isinstance(index, int):
raise TypeError("Not a valid array index: '{}'".format(index))
elif isinstance(obj, dict):
if not isinstance(index, (six.string_types, int)):
raise TypeError("Not a valid dictionary index: '{}'".format(index))
else:
raise TypeError("Object does not support indexing: '{}'".format(type(obj)))
return obj[index]
def test_map_reduce_workflow(self):
with open(os.path.join(TEST_DATA_DIR, "map_reduce_workflow.json"), "r") as f:
workflow = Workflow.from_json(f.read())
arr_1 = numpy.arange(10)
arr_2 = numpy.arange(10, 20)
arr_3 = numpy.arange(20, 30)
expected = sum([x / float(numpy.max(x)) for x in [arr_1, arr_2, arr_3]])
result = workflow(arrays_in=[arr_1, arr_2, arr_3])
array_out = result["array_out"]
assert is_ndarray(array_out)
assert (array_out == expected).all()
def test_reduce_by_expression(self):
task = ReduceByExpression()
arr_1 = numpy.arange(10)
arr_2 = numpy.arange(10, 20)
arr_3 = numpy.arange(20, 30)
expected = arr_1 + arr_2 + arr_3
expr = "x + y"
result = task(arrays_in=[arr_1, arr_2, arr_3], expression=expr)
array_out = result["array_out"]
assert is_ndarray(array_out)
assert (array_out == expected).all()
initial = numpy.arange(10)
expected += initial
result = task(arrays_in=[arr_1, arr_2, arr_3], expression=expr, initial_array=initial)
array_out = result["array_out"]
assert is_ndarray(array_out)
assert (array_out == expected).all()
result = task(arrays_in=[arr_1, arr_2, arr_3], expression="x + (y-z)", z=2)
assert (result["array_out"] == (arr_1 + (arr_2 - 2) + (arr_3 - 2))).all()
def test_map_reduce_workflow(self):
with open(os.path.join(TEST_DATA_DIR, 'map_reduce_workflow.json'),
'r') as f:
workflow = Workflow.from_json(f.read())
arr_1 = numpy.arange(10)
arr_2 = numpy.arange(10, 20)
arr_3 = numpy.arange(20, 30)
expected = sum(
[x / float(numpy.max(x)) for x in [arr_1, arr_2, arr_3]])
result = workflow(arrays_in=[arr_1, arr_2, arr_3])
array_out = result['array_out']
assert is_ndarray(array_out)
assert (array_out == expected).all()
def test_map_by_expression(self):
task = MapByExpression()
arr_1 = numpy.arange(10)
arr_2 = numpy.arange(10, 20)
expr = "x * 2"
result = task(arrays_in=[arr_1, arr_2], expression=expr)
arrays_out = result["arrays_out"]
assert all(is_ndarray(x) for x in arrays_out)
assert len(arrays_out) == 2
assert (arrays_out[0] == arr_1 * 2).all()
assert (arrays_out[1] == arr_2 * 2).all()
result = task(arrays_in=[arr_1, arr_2], expression="x * y", y=2)
arrays_out = result["arrays_out"]
assert (arrays_out[0] == arr_1 * 2).all()
assert (arrays_out[1] == arr_2 * 2).all()
result = task(arrays_in=[arr_1, arr_2], expression="x + i")
arrays_out = result["arrays_out"]
assert (arrays_out[0] == arr_1).all()
assert (arrays_out[1] == arr_2 + 1).all()
def test_map_by_expression(self):
task = MapByExpression()
arr_1 = numpy.arange(10)
arr_2 = numpy.arange(10, 20)
expr = 'x * 2'
result = task(arrays_in=[arr_1, arr_2], expression=expr)
arrays_out = result['arrays_out']
assert all(is_ndarray(x) for x in arrays_out)
assert len(arrays_out) == 2
assert (arrays_out[0] == arr_1 * 2).all()
assert (arrays_out[1] == arr_2 * 2).all()
result = task(arrays_in=[arr_1, arr_2], expression='x * y', y=2)
arrays_out = result['arrays_out']
assert (arrays_out[0] == arr_1 * 2).all()
assert (arrays_out[1] == arr_2 * 2).all()
result = task(arrays_in=[arr_1, arr_2], expression='x + i')
arrays_out = result['arrays_out']
assert (arrays_out[0] == arr_1).all()
assert (arrays_out[1] == arr_2 + 1).all()
def op_and(x, y):
if is_ndarray(x) and is_ndarray(y):
return x & y
return x and y
def test_is_ndarray():
assert is_ndarray(np.zeros((1,)))
assert not is_ndarray([0])
assert not is_ndarray((0,))
def test_is_ndarray():
assert is_ndarray(np.zeros((1,)))
assert is_ndarray([0]) == False
assert is_ndarray((0,)) == False
def op_or(x, y):
if is_ndarray(x) and is_ndarray(y):
return x | y
return x or y
def process_web_outputs(results,
job,
publish_raster_results=False,
renderer_or_fn=None):
outputs = results.format_args()
for k, v in iter(outputs.items()):
if is_raster(v) and publish_raster_results:
service_name = '{0}/{1}'.format(job.uuid, k)
rel_path = '{}.nc'.format(service_name)
abs_path = os.path.join(SERVICE_DATA_ROOT, rel_path)
os.makedirs(os.path.dirname(abs_path))
with Dataset(abs_path, 'w', format='NETCDF4') as ds:
if is_latlong(v.extent.projection):
x_var = 'longitude'
y_var = 'latitude'
else:
x_var = 'x'
y_var = 'y'
coord_vars = SpatialCoordinateVariables.from_bbox(
v.extent, *reversed(v.shape))
coord_vars.add_to_dataset(ds, x_var, y_var)
fill_value = v.fill_value if numpy.ma.core.is_masked(
v) else None
data_var = ds.createVariable('data',
v.dtype,
dimensions=(y_var, x_var),
fill_value=fill_value)
data_var[:] = v
set_crs(ds, 'data', v.extent.projection)
if callable(renderer_or_fn):
renderer = renderer_or_fn(v)
elif renderer_or_fn is None:
renderer = StretchedRenderer([
(numpy.min(v).item(), Color(0, 0, 0)),
(numpy.max(v).item(), Color(255, 255, 255))
])
else:
renderer = renderer_or_fn
with transaction.atomic():
service = Service.objects.create(
name=service_name,
description=
('This service has been automatically generated from the result of a geoprocessing job.'
),
data_path=rel_path,
projection=v.extent.projection.srs,
full_extent=v.extent,
initial_extent=v.extent,
)
Variable.objects.create(service=service,
index=0,
variable='data',
projection=v.extent.projection.srs,
x_dimension=x_var,
y_dimension=y_var,
name='data',
renderer=renderer,
full_extent=v.extent)
ProcessingResultService.objects.create(job=job,
service=service)
outputs[k] = service_name
elif is_ndarray(v):
if v.size < numpy.get_printoptions()['threshold']:
outputs[k] = v.tolist()
else:
outputs[k] = str(v)
return outputs
def test_is_ndarray():
assert is_ndarray(np.zeros((1, )))
assert is_ndarray([0]) == False
assert is_ndarray((0, )) == False
def op_or(x, y):
if is_ndarray(x) and is_ndarray(y):
return x | y
return x or y
def process_web_outputs(results, job, publish_raster_results=False, renderer_or_fn=None):
outputs = results.format_args()
for k, v in six.iteritems(outputs):
if is_raster(v) and publish_raster_results:
service_name = '{0}/{1}'.format(job.uuid, k)
rel_path = '{}.nc'.format(service_name)
abs_path = os.path.join(SERVICE_DATA_ROOT, rel_path)
os.makedirs(os.path.dirname(abs_path))
with Dataset(abs_path, 'w', format='NETCDF4') as ds:
if v.extent.projection.is_latlong():
x_var = 'longitude'
y_var = 'latitude'
else:
x_var = 'x'
y_var = 'y'
coord_vars = SpatialCoordinateVariables.from_bbox(v.extent, *reversed(v.shape))
coord_vars.add_to_dataset(ds, x_var, y_var)
fill_value = v.fill_value if is_masked(v) else None
data_var = ds.createVariable('data', v.dtype, dimensions=(y_var, x_var), fill_value=fill_value)
data_var[:] = v
set_crs(ds, 'data', v.extent.projection)
if callable(renderer_or_fn):
renderer = renderer_or_fn(v)
elif renderer_or_fn is None:
renderer = StretchedRenderer(
[(numpy.min(v).item(), Color(0, 0, 0)), (numpy.max(v).item(), Color(255, 255, 255))]
)
else:
renderer = renderer_or_fn
with transaction.atomic():
service = Service.objects.create(
name=service_name,
description='This service has been automatically generated from the result of a geoprocessing job.',
data_path=rel_path,
projection=v.extent.projection.srs,
full_extent=v.extent,
initial_extent=v.extent,
)
Variable.objects.create(
service=service,
index=0,
variable='data',
projection=v.extent.projection.srs,
x_dimension=x_var,
y_dimension=y_var,
name='data',
renderer=renderer,
full_extent=v.extent
)
ProcessingResultService.objects.create(job=job, service=service)
outputs[k] = service_name
elif is_ndarray(v):
if v.size < numpy.get_printoptions()['threshold']:
outputs[k] = v.tolist()
else:
outputs[k] = str(v)
return outputs
def op_and(x, y):
if is_ndarray(x) and is_ndarray(y):
return x & y
return x and y