def test_reduce_by_expression(self): task = ReduceByExpression() arr_1 = numpy.arange(10) arr_2 = numpy.arange(10, 20) arr_3 = numpy.arange(20, 30) expected = arr_1 + arr_2 + arr_3 expr = 'x + y' result = task(arrays_in=[arr_1, arr_2, arr_3], expression=expr) array_out = result['array_out'] assert is_ndarray(array_out) assert (array_out == expected).all() initial = numpy.arange(10) expected += initial result = task(arrays_in=[arr_1, arr_2, arr_3], expression=expr, initial_array=initial) array_out = result['array_out'] assert is_ndarray(array_out) assert (array_out == expected).all() result = task(arrays_in=[arr_1, arr_2, arr_3], expression='x + (y-z)', z=2) assert (result['array_out'] == (arr_1 + (arr_2 - 2) + (arr_3 - 2))).all()
def fillnodata( image, mask=None, max_search_distance=100.0, smoothing_iterations=0): """Fill holes in raster data by interpolation This algorithm will interpolate values for all designated nodata pixels (marked by zeros in `mask`). For each pixel a four direction conic search is done to find values to interpolate from (using inverse distance weighting). Once all values are interpolated, zero or more smoothing iterations (3x3 average filters on interpolated pixels) are applied to smooth out artifacts. This algorithm is generally suitable for interpolating missing regions of fairly continuously varying rasters (such as elevation models for instance). It is also suitable for filling small holes and cracks in more irregularly varying images (like aerial photos). It is generally not so great for interpolating a raster from sparse point data. Parameters ---------- image : numpy ndarray The source image with holes to be filled. If a MaskedArray, the inverse of its mask will define the pixels to be filled -- unless the ``mask`` argument is not None (see below).` mask : numpy ndarray or None A mask band indicating which pixels to interpolate. Pixels to interpolate into are indicated by the value 0. Values > 0 indicate areas to use during interpolation. Must be same shape as image. This array always takes precedence over the image's mask (see above). If None, the inverse of the image's mask will be used if available. max_search_distance : float, optional The maxmimum number of pixels to search in all directions to find values to interpolate from. The default is 100. smoothing_iterations : integer, optional The number of 3x3 smoothing filter passes to run. The default is 0. Returns ------- out : numpy ndarray The filled raster array. """ if mask is None and isinstance(image, MaskedArray): mask = ~image.mask if not dtypes.is_ndarray(mask): raise ValueError("An mask array is required") if isinstance(image, MaskedArray): image = image.data if not dtypes.is_ndarray(image): raise ValueError("An image array is required") max_search_distance = float(max_search_distance) smoothing_iterations = int(smoothing_iterations) return _fillnodata( image, mask, max_search_distance, smoothing_iterations)
def evaluate_expression(self, expression, context={}): try: # Operations against masked arrays are really slow, so take a regular array view, then back to a masked # array afterwards. Todo: find a better solution long-term expr_context = {k: v.view(numpy.ndarray) if is_masked(v) else v for k, v in six.iteritems(context)} result = Parser().evaluate(expression, context=expr_context) if is_ndarray(result): for value in six.itervalues(context): if is_masked(value): if is_masked(result) and is_masked(value): result.mask = result.mask | value.mask elif is_masked(value): result = numpy.ma.masked_array(result, mask=value.mask) result = Raster(result, value.extent, value.x_dim, value.y_dim, value.y_increasing) break return result except (SyntaxError, NameError) as e: raise ExecutionError( 'The expression is invalid ({0}): {1}\nContext: {2}'.format(str(e), expression, str(context)), self )
def evaluate_expression(self, expression, context={}): try: # Operations against masked arrays are really slow, so take a regular array view, then back to a masked # array afterwards. Todo: find a better solution long-term expr_context = { k: v.view(ndarray) if is_masked(v) else v for k, v in context.items() } result = Parser().evaluate(expression, context=expr_context) if is_ndarray(result): for value in context.values(): if is_masked(value): if is_masked(result) and is_masked(value): result.mask = result.mask | value.mask elif is_masked(value): result = masked_array(result, mask=value.mask) result = Raster(result, value.extent, value.x_dim, value.y_dim, value.y_increasing) break return result except (SyntaxError, NameError) as e: raise ExecutionError( 'The expression is invalid ({0}): {1}\nContext: {2}'.format( str(e), expression, str(context)), self)
def test_array_from_dataset(self): task = ArrayFromDataset() dataset = Dataset(os.path.join(TEST_DATA_DIR, "simple_grid_2d.nc")) result = task(dataset=dataset, variable="value") arr = result["array_out"] assert is_ndarray(arr) assert arr.shape == (10, 10) assert (arr == numpy.reshape(numpy.arange(100), (10, 10))).all()
def test_array_from_dataset(self): task = ArrayFromDataset() dataset = Dataset(os.path.join(TEST_DATA_DIR, 'simple_grid_2d.nc')) result = task(dataset=dataset, variable='value') arr = result['array_out'] assert is_ndarray(arr) assert arr.shape == (10, 10) assert (arr == numpy.reshape(numpy.arange(100), (10, 10))).all()
def test_raster(self): arr = numpy.reshape(numpy.arange(100), (10, 10)) extent = BBox((0, 0, 10, 10)) raster = Raster(arr, extent, 1, 0) assert isinstance(raster, Raster) raster = raster[:] assert isinstance(raster, Raster) assert isinstance(raster.extent, BBox) clipped = raster[3:, 3:-1] assert isinstance(clipped, Raster) assert isinstance(raster.extent, BBox) assert clipped.extent.as_list() == [3, 0, 10, 7] raster.y_increasing = True clipped = raster[3:, 3:-1] assert isinstance(clipped, Raster) assert isinstance(raster.extent, BBox) assert clipped.extent.as_list() == [3, 3, 10, 10] clipped = raster[0] assert not isinstance(clipped, Raster) assert isinstance(raster.extent, BBox) assert is_ndarray(clipped) assert (clipped == numpy.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])).all() # Make sure binops don't lose extent info assert isinstance((raster == raster).extent, BBox) assert isinstance((raster != raster).extent, BBox) assert isinstance((raster > 2).extent, BBox) assert isinstance((2 > raster).extent, BBox) assert isinstance((raster + 2).extent, BBox) assert isinstance((2 + raster).extent, BBox) assert isinstance((raster - 2).extent, BBox) assert isinstance((2 - raster).extent, BBox) assert isinstance((raster * 2).extent, BBox) assert isinstance((2 * raster).extent, BBox) assert isinstance((raster / 2).extent, BBox) assert isinstance((2 / raster).extent, BBox) assert isinstance((raster // 2).extent, BBox) assert isinstance((2 // raster).extent, BBox) assert isinstance((raster ** 2).extent, BBox) assert isinstance((2 ** raster).extent, BBox) assert isinstance((raster % 2).extent, BBox) assert isinstance((2 % raster).extent, BBox) assert isinstance((raster << 2).extent, BBox) assert isinstance((2 << raster).extent, BBox) assert isinstance((raster >> 2).extent, BBox) assert isinstance((2 >> raster).extent, BBox) assert isinstance((raster | 2).extent, BBox) assert isinstance((2 | raster).extent, BBox) assert isinstance((raster & 2).extent, BBox) assert isinstance((2 & raster).extent, BBox) assert isinstance((raster ^ 2).extent, BBox) assert isinstance((2 ^ raster).extent, BBox)
def test_raster(self): arr = numpy.reshape(numpy.arange(100), (10, 10)) extent = BBox((0, 0, 10, 10)) raster = Raster(arr, extent, 1, 0) assert isinstance(raster, Raster) raster = raster[:] assert isinstance(raster, Raster) assert isinstance(raster.extent, BBox) clipped = raster[3:, 3:-1] assert isinstance(clipped, Raster) assert isinstance(raster.extent, BBox) assert clipped.extent.as_list() == [3, 0, 10, 7] raster.y_increasing = True clipped = raster[3:, 3:-1] assert isinstance(clipped, Raster) assert isinstance(raster.extent, BBox) assert clipped.extent.as_list() == [3, 3, 10, 10] clipped = raster[0] assert not isinstance(clipped, Raster) assert isinstance(raster.extent, BBox) assert is_ndarray(clipped) assert (clipped == numpy.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])).all() # Make sure binops don't lose extent info assert isinstance((raster == raster).extent, BBox) assert isinstance((raster != raster).extent, BBox) assert isinstance((raster > 2).extent, BBox) assert isinstance((2 > raster).extent, BBox) assert isinstance((raster + 2).extent, BBox) assert isinstance((2 + raster).extent, BBox) assert isinstance((raster - 2).extent, BBox) assert isinstance((2 - raster).extent, BBox) assert isinstance((raster * 2).extent, BBox) assert isinstance((2 * raster).extent, BBox) assert isinstance((raster / 2).extent, BBox) assert isinstance((2 / raster).extent, BBox) assert isinstance((raster // 2).extent, BBox) assert isinstance((2 // raster).extent, BBox) assert isinstance((raster**2).extent, BBox) assert isinstance((2**raster).extent, BBox) assert isinstance((raster % 2).extent, BBox) assert isinstance((2 % raster).extent, BBox) assert isinstance((raster << 2).extent, BBox) assert isinstance((2 << raster).extent, BBox) assert isinstance((raster >> 2).extent, BBox) assert isinstance((2 >> raster).extent, BBox) assert isinstance((raster | 2).extent, BBox) assert isinstance((2 | raster).extent, BBox) assert isinstance((raster & 2).extent, BBox) assert isinstance((2 & raster).extent, BBox) assert isinstance((raster ^ 2).extent, BBox) assert isinstance((2 ^ raster).extent, BBox)
def _to_ndarray(self, a): """Casts Python lists and tuples to a numpy array or raises an AssertionError.""" if isinstance(a, (list, tuple)): a = numpy.array(a) if not is_ndarray(a): raise TypeError("Expected an ndarray but got object of type '{}' instead".format(type(a))) return a
def test_apply_expression(self): task = ApplyExpression() arr = numpy.array([1, 2, 3]) expr = "(x ** 2) / 2" result = task(array_in=arr, expression=expr) assert is_ndarray(result["array_out"]) assert (result["array_out"] == numpy.array([0.5, 2, 4.5])).all() result = task(array_in=arr, expression="(x ** y) / y", y=2) assert (result["array_out"] == numpy.array([0.5, 2, 4.5])).all()
def test_apply_expression(self): task = ApplyExpression() arr = numpy.array([1, 2, 3]) expr = '(x ** 2) / 2' result = task(array_in=arr, expression=expr) assert is_ndarray(result['array_out']) assert (result['array_out'] == numpy.array([.5, 2, 4.5])).all() result = task(array_in=arr, expression='(x ** y) / y', y=2) assert (result['array_out'] == numpy.array([.5, 2, 4.5])).all()
def fn_abs(self, value): """ Return the absolute value of a number. :param value: The number. :return: The absolute value of the number. """ if is_ndarray(value): return numpy.absolute(value) else: return abs(value)
def fn_int32(self, value): """ Return the value cast to an 64-bit signed integer (numpy array) or a Python int (single value) :param value: The number or array :return: The number or array as int/int8 """ if is_ndarray(value) or isinstance(value, (list, tuple)): return self._to_ndarray(value).astype(numpy.int64) else: return int(value)
def fn_ceil(self, value): """ Return the ceiling of a number. :param value: The number. :return: The ceiling of the number. """ if is_ndarray(value) or isinstance(value, (list, tuple)): return numpy.ceil(self._to_ndarray(value)) else: return math.ceil(value)
def fn_floor(self, value): """ Return the floor of a number. For negative numbers, floor returns a lower value. E.g., `floor(-2.5) == -3` :param value: The number. :return: The floor of the number. """ if is_ndarray(value) or isinstance(value, (list, tuple)): return numpy.floor(self._to_ndarray(value)) else: return math.floor(value)
def fn_float64(self, value): """ Return the value cast to a 64-bit float (numpy array) or a Python float (single value). :param value: The number. :return: The number as a float. """ if is_ndarray(value) or isinstance(value, (list, tuple)): return self._to_ndarray(value).astype('float64') else: return float(value)
def fn_int64(self, value): """ Return the value cast to an 64-bit signed integer (numpy array) or a Python int (single value) :param value: The number or array :return: The number or array as int/int8 """ if is_ndarray(value) or isinstance(value, (list, tuple)): return self._to_ndarray(value).astype(numpy.int64) else: return int(value)
def fn_float(self, value): """ Return the value cast to a float. :param value: The number. :return: The number as a float. """ if is_ndarray(value) or isinstance(value, (list, tuple)): return self._to_ndarray(value).astype('float') else: return float(value)
def resolve_item(obj, index): if is_ndarray(obj) or isinstance(obj, (list, tuple)): if not isinstance(index, int): raise TypeError("Not a valid array index: '{}'".format(index)) elif isinstance(obj, dict): if not isinstance(index, (str, int)): raise TypeError("Not a valid dictionary index: '{}'".format(index)) else: raise TypeError("Object does not support indexing: '{}'".format(type(obj))) return obj[index]
def resolve_item(obj, index): if is_ndarray(obj) or isinstance(obj, (list, tuple)): if not isinstance(index, int): raise TypeError("Not a valid array index: '{}'".format(index)) elif isinstance(obj, dict): if not isinstance(index, (six.string_types, int)): raise TypeError("Not a valid dictionary index: '{}'".format(index)) else: raise TypeError("Object does not support indexing: '{}'".format(type(obj))) return obj[index]
def test_map_reduce_workflow(self): with open(os.path.join(TEST_DATA_DIR, "map_reduce_workflow.json"), "r") as f: workflow = Workflow.from_json(f.read()) arr_1 = numpy.arange(10) arr_2 = numpy.arange(10, 20) arr_3 = numpy.arange(20, 30) expected = sum([x / float(numpy.max(x)) for x in [arr_1, arr_2, arr_3]]) result = workflow(arrays_in=[arr_1, arr_2, arr_3]) array_out = result["array_out"] assert is_ndarray(array_out) assert (array_out == expected).all()
def test_reduce_by_expression(self): task = ReduceByExpression() arr_1 = numpy.arange(10) arr_2 = numpy.arange(10, 20) arr_3 = numpy.arange(20, 30) expected = arr_1 + arr_2 + arr_3 expr = "x + y" result = task(arrays_in=[arr_1, arr_2, arr_3], expression=expr) array_out = result["array_out"] assert is_ndarray(array_out) assert (array_out == expected).all() initial = numpy.arange(10) expected += initial result = task(arrays_in=[arr_1, arr_2, arr_3], expression=expr, initial_array=initial) array_out = result["array_out"] assert is_ndarray(array_out) assert (array_out == expected).all() result = task(arrays_in=[arr_1, arr_2, arr_3], expression="x + (y-z)", z=2) assert (result["array_out"] == (arr_1 + (arr_2 - 2) + (arr_3 - 2))).all()
def test_map_reduce_workflow(self): with open(os.path.join(TEST_DATA_DIR, 'map_reduce_workflow.json'), 'r') as f: workflow = Workflow.from_json(f.read()) arr_1 = numpy.arange(10) arr_2 = numpy.arange(10, 20) arr_3 = numpy.arange(20, 30) expected = sum( [x / float(numpy.max(x)) for x in [arr_1, arr_2, arr_3]]) result = workflow(arrays_in=[arr_1, arr_2, arr_3]) array_out = result['array_out'] assert is_ndarray(array_out) assert (array_out == expected).all()
def test_map_by_expression(self): task = MapByExpression() arr_1 = numpy.arange(10) arr_2 = numpy.arange(10, 20) expr = "x * 2" result = task(arrays_in=[arr_1, arr_2], expression=expr) arrays_out = result["arrays_out"] assert all(is_ndarray(x) for x in arrays_out) assert len(arrays_out) == 2 assert (arrays_out[0] == arr_1 * 2).all() assert (arrays_out[1] == arr_2 * 2).all() result = task(arrays_in=[arr_1, arr_2], expression="x * y", y=2) arrays_out = result["arrays_out"] assert (arrays_out[0] == arr_1 * 2).all() assert (arrays_out[1] == arr_2 * 2).all() result = task(arrays_in=[arr_1, arr_2], expression="x + i") arrays_out = result["arrays_out"] assert (arrays_out[0] == arr_1).all() assert (arrays_out[1] == arr_2 + 1).all()
def test_map_by_expression(self): task = MapByExpression() arr_1 = numpy.arange(10) arr_2 = numpy.arange(10, 20) expr = 'x * 2' result = task(arrays_in=[arr_1, arr_2], expression=expr) arrays_out = result['arrays_out'] assert all(is_ndarray(x) for x in arrays_out) assert len(arrays_out) == 2 assert (arrays_out[0] == arr_1 * 2).all() assert (arrays_out[1] == arr_2 * 2).all() result = task(arrays_in=[arr_1, arr_2], expression='x * y', y=2) arrays_out = result['arrays_out'] assert (arrays_out[0] == arr_1 * 2).all() assert (arrays_out[1] == arr_2 * 2).all() result = task(arrays_in=[arr_1, arr_2], expression='x + i') arrays_out = result['arrays_out'] assert (arrays_out[0] == arr_1).all() assert (arrays_out[1] == arr_2 + 1).all()
def op_and(x, y): if is_ndarray(x) and is_ndarray(y): return x & y return x and y
def test_is_ndarray(): assert is_ndarray(np.zeros((1,))) assert not is_ndarray([0]) assert not is_ndarray((0,))
def test_is_ndarray(): assert is_ndarray(np.zeros((1,))) assert is_ndarray([0]) == False assert is_ndarray((0,)) == False
def op_or(x, y): if is_ndarray(x) and is_ndarray(y): return x | y return x or y
def process_web_outputs(results, job, publish_raster_results=False, renderer_or_fn=None): outputs = results.format_args() for k, v in iter(outputs.items()): if is_raster(v) and publish_raster_results: service_name = '{0}/{1}'.format(job.uuid, k) rel_path = '{}.nc'.format(service_name) abs_path = os.path.join(SERVICE_DATA_ROOT, rel_path) os.makedirs(os.path.dirname(abs_path)) with Dataset(abs_path, 'w', format='NETCDF4') as ds: if is_latlong(v.extent.projection): x_var = 'longitude' y_var = 'latitude' else: x_var = 'x' y_var = 'y' coord_vars = SpatialCoordinateVariables.from_bbox( v.extent, *reversed(v.shape)) coord_vars.add_to_dataset(ds, x_var, y_var) fill_value = v.fill_value if numpy.ma.core.is_masked( v) else None data_var = ds.createVariable('data', v.dtype, dimensions=(y_var, x_var), fill_value=fill_value) data_var[:] = v set_crs(ds, 'data', v.extent.projection) if callable(renderer_or_fn): renderer = renderer_or_fn(v) elif renderer_or_fn is None: renderer = StretchedRenderer([ (numpy.min(v).item(), Color(0, 0, 0)), (numpy.max(v).item(), Color(255, 255, 255)) ]) else: renderer = renderer_or_fn with transaction.atomic(): service = Service.objects.create( name=service_name, description= ('This service has been automatically generated from the result of a geoprocessing job.' ), data_path=rel_path, projection=v.extent.projection.srs, full_extent=v.extent, initial_extent=v.extent, ) Variable.objects.create(service=service, index=0, variable='data', projection=v.extent.projection.srs, x_dimension=x_var, y_dimension=y_var, name='data', renderer=renderer, full_extent=v.extent) ProcessingResultService.objects.create(job=job, service=service) outputs[k] = service_name elif is_ndarray(v): if v.size < numpy.get_printoptions()['threshold']: outputs[k] = v.tolist() else: outputs[k] = str(v) return outputs
def test_is_ndarray(): assert is_ndarray(np.zeros((1, ))) assert is_ndarray([0]) == False assert is_ndarray((0, )) == False
def process_web_outputs(results, job, publish_raster_results=False, renderer_or_fn=None): outputs = results.format_args() for k, v in six.iteritems(outputs): if is_raster(v) and publish_raster_results: service_name = '{0}/{1}'.format(job.uuid, k) rel_path = '{}.nc'.format(service_name) abs_path = os.path.join(SERVICE_DATA_ROOT, rel_path) os.makedirs(os.path.dirname(abs_path)) with Dataset(abs_path, 'w', format='NETCDF4') as ds: if v.extent.projection.is_latlong(): x_var = 'longitude' y_var = 'latitude' else: x_var = 'x' y_var = 'y' coord_vars = SpatialCoordinateVariables.from_bbox(v.extent, *reversed(v.shape)) coord_vars.add_to_dataset(ds, x_var, y_var) fill_value = v.fill_value if is_masked(v) else None data_var = ds.createVariable('data', v.dtype, dimensions=(y_var, x_var), fill_value=fill_value) data_var[:] = v set_crs(ds, 'data', v.extent.projection) if callable(renderer_or_fn): renderer = renderer_or_fn(v) elif renderer_or_fn is None: renderer = StretchedRenderer( [(numpy.min(v).item(), Color(0, 0, 0)), (numpy.max(v).item(), Color(255, 255, 255))] ) else: renderer = renderer_or_fn with transaction.atomic(): service = Service.objects.create( name=service_name, description='This service has been automatically generated from the result of a geoprocessing job.', data_path=rel_path, projection=v.extent.projection.srs, full_extent=v.extent, initial_extent=v.extent, ) Variable.objects.create( service=service, index=0, variable='data', projection=v.extent.projection.srs, x_dimension=x_var, y_dimension=y_var, name='data', renderer=renderer, full_extent=v.extent ) ProcessingResultService.objects.create(job=job, service=service) outputs[k] = service_name elif is_ndarray(v): if v.size < numpy.get_printoptions()['threshold']: outputs[k] = v.tolist() else: outputs[k] = str(v) return outputs