def evaluate_expression(self, expression, context={}):
try:
# Operations against masked arrays are really slow, so take a regular array view, then back to a masked
# array afterwards. Todo: find a better solution long-term
expr_context = {
k: v.view(ndarray) if is_masked(v) else v
for k, v in context.items()
}
result = Parser().evaluate(expression, context=expr_context)
if is_ndarray(result):
for value in context.values():
if is_masked(value):
if is_masked(result) and is_masked(value):
result.mask = result.mask | value.mask
elif is_masked(value):
result = masked_array(result, mask=value.mask)
result = Raster(result, value.extent, value.x_dim,
value.y_dim, value.y_increasing)
break
return result
except (SyntaxError, NameError) as e:
raise ExecutionError(
'The expression is invalid ({0}): {1}\nContext: {2}'.format(
str(e), expression, str(context)), self)
def evaluate_expression(self, expression, context={}):
try:
# Operations against masked arrays are really slow, so take a regular array view, then back to a masked
# array afterwards. Todo: find a better solution long-term
expr_context = {k: v.view(numpy.ndarray) if is_masked(v) else v for k, v in six.iteritems(context)}
result = Parser().evaluate(expression, context=expr_context)
if is_ndarray(result):
for value in six.itervalues(context):
if is_masked(value):
if is_masked(result) and is_masked(value):
result.mask = result.mask | value.mask
elif is_masked(value):
result = numpy.ma.masked_array(result, mask=value.mask)
result = Raster(result, value.extent, value.x_dim, value.y_dim, value.y_increasing)
break
return result
except (SyntaxError, NameError) as e:
raise ExecutionError(
'The expression is invalid ({0}): {1}\nContext: {2}'.format(str(e), expression, str(context)),
self
)
def to_paletted_png(arr, palette, nodata=None):
"""
Render an array as a paletted PNG.
Parameters
----------
arr : input array or masked array, must have dtype of uint8
palette : numpy array of 8 bit tuples [(r, g, b), ...], where the index corresponds to the value in the image
nodata : nodata value, will be set as transparent in the image (optional, default: None)
Returns
-------
PNG bytes
"""
if arr.dtype.kind not in ("u", "i"):
raise ValueError("Input array must be integer type")
# TODO: validate palette: must be of a small enough size and have rgb tuples
nodata_index = None
if is_masked(arr) or nodata is not None:
# If it is masked, fill with index at end of palette
nodata_index = len(palette)
# add nodata color to palette (set as transparent below)
palette = np.append(palette, (0, 0, 0))
if is_masked(arr):
arr = arr.filled(nodata_index)
else:
arr[arr == nodata] = nodata_index
else:
# return the underlying ndarray
arr = arr.data
img = Image.frombuffer("P", (arr.shape[1], arr.shape[0]), arr, "raw", "P",
0, 1)
# palette must be a list of [r, g, b, r, g, b, ...] values
img.putpalette(palette.flatten().tolist(), "RGB")
if nodata_index is not None:
img.info["transparency"] = nodata_index
buf = BytesIO()
img.save(buf, "PNG")
buf.seek(0) # rewind to beginning of buffer
return buf.read()
def handle_request(self, request, **kwargs):
try:
configurations = self.get_identify_configurations(request, **kwargs)
if not configurations:
return HttpResponse()
data = {}
for config in configurations:
variable = config.variable
service = variable.service
if service.supports_time and variable.supports_time:
time_index = config.time_index or 0
else:
time_index = None
geometry = project_geometry(
config.geometry, config.projection, pyproj.Proj(str(self.service.projection))
)
assert isinstance(geometry, Point) # Only point-based identify is supported
dimensions = self.get_grid_spatial_dimensions(config.variable)
cell_size = (
float(variable.full_extent.width) / dimensions[0],
float(variable.full_extent.height) / dimensions[1]
)
cell_index = [
int(float(geometry.x-variable.full_extent.xmin) / cell_size[0]),
int(float(geometry.y-variable.full_extent.ymin) / cell_size[1])
]
if not self.is_y_increasing(variable):
cell_index[1] = dimensions[1] - cell_index[1] - 1
variable_data = self.get_grid_for_variable(
config.variable, time_index=time_index, x_slice=(cell_index[0], cell_index[0] + 1),
y_slice=(cell_index[1], cell_index[1] + 1)
)
if len(variable_data):
value = variable_data[0][0]
data[variable] = None if is_masked(value) else float(value)
data, content_type = self.serialize_data(data)
return self.create_response(request, data, content_type=content_type)
except ConfigurationError:
return HttpResponseBadRequest()
finally:
self.close_dataset()
def get_queryset(self):
if not self.request.query_params.get('point'):
return self.queryset
else:
try:
x, y = [float(x) for x in self.request.query_params['point'].split(',')]
except ValueError:
raise ParseError()
elevation = get_elevation_at_point(Point(x, y))
if elevation is None or is_masked(elevation):
return self.queryset.none()
# Elevation bands are stored in feet
return self.queryset.filter(
Q(low__lt=elevation/0.3048, high__gte=elevation/0.3048) | Q(low__isnull=True, high__isnull=True)
)
def __new__(cls, arr, extent, x_dim, y_dim, y_increasing=False):
"""Create a new Raster from a numpy array and a `BBox` object."""
assert len(arr.shape) > 1
assert isinstance(extent, BBox)
obj = numpy.asarray(arr).view(cls)
obj.x_dim = x_dim
obj.y_dim = y_dim
obj.extent = extent
obj.y_increasing = y_increasing
obj.__array_priority__ = 100
if is_masked(arr):
obj._mask = arr._mask
obj._fill_value = arr._fill_value
return obj
def get_queryset(self):
if not self.request.query_params.get('point'):
return self.queryset
else:
try:
x, y = [
float(x)
for x in self.request.query_params['point'].split(',')
]
except ValueError:
raise ParseError()
elevation = get_elevation_at_point(Point(x, y))
if elevation is None or is_masked(elevation):
return self.queryset.none()
# Elevation bands are stored in feet
return self.queryset.filter(
Q(low__lt=elevation / 0.3048, high__gte=elevation / 0.3048)
| Q(low__isnull=True, high__isnull=True))
def process_web_outputs(results, job, publish_raster_results=False, renderer_or_fn=None):
outputs = results.format_args()
for k, v in six.iteritems(outputs):
if is_raster(v) and publish_raster_results:
service_name = '{0}/{1}'.format(job.uuid, k)
rel_path = '{}.nc'.format(service_name)
abs_path = os.path.join(SERVICE_DATA_ROOT, rel_path)
os.makedirs(os.path.dirname(abs_path))
with Dataset(abs_path, 'w', format='NETCDF4') as ds:
if v.extent.projection.is_latlong():
x_var = 'longitude'
y_var = 'latitude'
else:
x_var = 'x'
y_var = 'y'
coord_vars = SpatialCoordinateVariables.from_bbox(v.extent, *reversed(v.shape))
coord_vars.add_to_dataset(ds, x_var, y_var)
fill_value = v.fill_value if is_masked(v) else None
data_var = ds.createVariable('data', v.dtype, dimensions=(y_var, x_var), fill_value=fill_value)
data_var[:] = v
set_crs(ds, 'data', v.extent.projection)
if callable(renderer_or_fn):
renderer = renderer_or_fn(v)
elif renderer_or_fn is None:
renderer = StretchedRenderer(
[(numpy.min(v).item(), Color(0, 0, 0)), (numpy.max(v).item(), Color(255, 255, 255))]
)
else:
renderer = renderer_or_fn
with transaction.atomic():
service = Service.objects.create(
name=service_name,
description='This service has been automatically generated from the result of a geoprocessing job.',
data_path=rel_path,
projection=v.extent.projection.srs,
full_extent=v.extent,
initial_extent=v.extent,
)
Variable.objects.create(
service=service,
index=0,
variable='data',
projection=v.extent.projection.srs,
x_dimension=x_var,
y_dimension=y_var,
name='data',
renderer=renderer,
full_extent=v.extent
)
ProcessingResultService.objects.create(job=job, service=service)
outputs[k] = service_name
elif is_ndarray(v):
if v.size < numpy.get_printoptions()['threshold']:
outputs[k] = v.tolist()
else:
outputs[k] = str(v)
return outputs
def to_smallest_png(arr, image_type=None):
"""
Convert an array to PNG, using the smallest PNG bit depth that
will contain the data range: 8, 24, or 32.
If the input is a masked array, the maximum value of the data type
will be used to fill nodata.
You can pre-fill nodata with a different value, but if you use a value well
outside your value range, this may force use of a larger output PNG bit depth
than is ideal.
Parameters
----------
arr: input array or masked array, must have dtype of uint8, uint16, or uint32
Returns
-------
PNG bytes
"""
if arr.dtype.kind not in ("u", "i"):
raise ValueError("Input array must be integer type")
if image_type is None:
image_type = get_smallest_image_type(arr)
else:
if not image_type in ("L", "RGB", "RGBA"):
raise ValueError("Image type must be one of: L, RGB, RGBA")
# Temporary: either remove RGBA support or make it work in browsers
if image_type == "RGBA":
raise ValueError(
"RGBA is not currently supported due to variable decoding in browsers"
)
if is_masked(arr):
# If it is masked, fill it with appropriate nodata value
image_type_max = MAX_VALUE[image_type]
if arr.max() < image_type_max:
arr = arr.filled(image_type_max)
else:
raise ValueError("Max of value range must be max of data type - 1"
"to allow max of data type to be nodata value")
else:
# return the underlying ndarray
arr = arr.data
if image_type == "L":
image_data = arr
elif image_type == "RGB":
image_data = to_rgb_array(np.asarray(arr))
elif image_type == "RGBA":
image_data = to_rgba_array(np.asarray(arr))
else:
raise NotImplementedError(
"values require an image type that is not supported")
img = Image.frombuffer(image_type, (arr.shape[1], arr.shape[0]),
image_data, "raw", image_type, 0, 1)
buf = BytesIO()
img.save(buf, "PNG")
buf.seek(0) # rewind to beginning of buffer
return buf.read()
def main(in_pattern, out_pattern, boundary, single, varname):
"""
Clips and masks large NetCDF datasets to regional datasets based on the boundary. The in_pattern and out_pattern
arguments should be filename patterns (can include path) with the pattern: /path/to/in_netcdf_{variable}.nc.
Example usage: python cut_to_region.py NorthAmerica/NA_{variable}.nc USWest/west_{variable}.nc west.shp
"""
if single and not varname:
print('--varname is required when --single is used')
sys.exit(-1)
if single:
if not os.path.exists(in_pattern):
print('Input file {} does not exist.'.format(in_pattern))
sys.exit(-1)
input_paths = [(in_pattern, varname)]
else:
input_paths = [(in_pattern.format(variable=x), x) for x in VARIABLES]
for path, _ in input_paths:
if not os.path.exists(path):
print('Input file {} does not exist.'.format(path))
sys.exit(-1)
with fiona.open(boundary, 'r') as shp:
features = []
wgs84 = Proj('+init=EPSG:4326')
shp_projection = Proj(shp.crs)
bounds = shp.bounds
ll = transform(shp_projection, wgs84, bounds[0], bounds[1])
ur = transform(shp_projection, wgs84, bounds[2], bounds[3])
bbox = BBox([*ll, *ur], projection=wgs84)
for feature in shp.items():
geometry = transform_geom(shp.crs, {'init': 'EPSG: 4326'},
feature[1]['geometry'])
features.append(geometry)
for in_path, variable in input_paths:
if single:
out_path = out_pattern
else:
out_path = out_pattern.format(variable=variable)
if os.path.exists(out_path):
confirm = input(
"The output file '{}' already exists? Do you with to replace it? [y/n] "
.format(out_path))
if confirm.lower().strip() not in ['y', 'yes']:
print('Exiting...')
sys.exit()
with Dataset(in_path, 'r') as ds:
coords = SpatialCoordinateVariables.from_dataset(
ds, x_name='longitude', y_name='latitude')
x_start, x_stop = coords.x.indices_for_range(bbox.xmin, bbox.xmax)
y_start, y_stop = coords.y.indices_for_range(bbox.ymin, bbox.ymax)
x_slice = slice(x_start, x_stop)
y_slice = slice(y_start, y_stop)
clipped_coords = coords.slice_by_bbox(bbox)
grid = ds.variables[variable][y_slice, x_slice]
if is_masked(grid):
mask = grid.mask.astype('uint8')
else:
mask = numpy.zeros(grid.shape, dtype='uint8')
mask |= rasterize(((x, 0) for x in features),
out_shape=mask.shape,
transform=clipped_coords.affine,
fill=1,
default_value=0)
grid = numpy.ma.masked_where(mask == 1, grid.data)
print('Writing {}...'.format(out_path))
with Dataset(out_path, 'w', format='NETCDF4') as ds:
clipped_coords.add_to_dataset(ds, 'longitude', 'latitude')
data_var = ds.createVariable(variable,
grid.dtype,
dimensions=('latitude', 'longitude'),
fill_value=grid.fill_value)
if data_var.shape != grid.shape:
grid = grid[:data_var.shape[0], :data_var.shape[1]]
data_var[:] = grid
set_crs(ds, variable, Proj('+init=EPSG:4326'))