def update_grids_bbox(self, bbox):
"""Update quad_grid_dps_mask using bbox
bbox format: [x0, x1, y0, y1]
"""
if 'quad_grid_dps_mask' not in self.grid:
logger.debug(
"Calling update_grids instead of update_grids_bbox.",
extra={'subgrid_id': self.subgrid_id})
return self.update_grids()
x0, x1, y0, y1 = bbox
quad_grid = self.grid['quad_grid'][y0:y1, x0:x1]
dps = self.grid['dps'][y0:y1, x0:x1]
logger.debug(
'quad grid bbox shape: %r' % (str(quad_grid.shape)),
extra={'subgrid_id': self.subgrid_id})
logger.debug(
'dps bbox shape: %r' % (str(dps.shape)),
extra={'subgrid_id': self.subgrid_id})
# Sometimes quad_grid.mask is False instead of a table... (model Miami)
# TODO: investigate more
if quad_grid.mask.__class__.__name__ == 'bool_':
mask = utils.logical_or_reduce([dps < -9000, ])
else:
mask = utils.logical_or_reduce([quad_grid.mask, dps < -9000])
self.grid['quad_grid_dps_mask'][y0:y1, x0:x1] = mask
def update_grids(self):
"""Precalculate quad_grid_dps_mask that only needs to be calculated once.
Needs to be run when quad_grid or dps is updated.
"""
grid = self.grid
quad_grid = grid['quad_grid']
dps = grid['dps']
logger.debug(
'quad grid shape: %r' % (str(quad_grid.shape)),
extra={'subgrid_id': self.subgrid_id})
logger.debug(
'dps shape: %r' % (str(dps.shape)),
extra={'subgrid_id': self.subgrid_id})
# Sometimes quad_grid.mask is False instead of a table... (model Miami)
# TODO: investigate more
if quad_grid.mask.__class__.__name__ == 'bool_':
mask = utils.logical_or_reduce([dps < -9000, ])
else:
# 4 seconds
mask = utils.logical_or_reduce([quad_grid.mask, dps < -9000])
if 'quad_grid_dps_mask' in grid:
del self.grid['quad_grid_dps_mask']
self.grid['quad_grid_dps_mask'] = mask
def get(self, layer, interpolate='nearest', from_disk=False, **kwargs):
"""
layer: choose from waterlevel, waterheight, dps, uc,
sg, quad_grid, infiltration,
interception, soil, crop, maxdepth, arrival
from_disk: read grids.nc instead of memory, kwargs must contain kw
'layers': duifp-duifp:maxdepth
NOTE: maxdepth and arrival REQUIRE the from_disk method.
"""
def generate_hash(path, layer_slug):
return '%r-%r-%r' % (
layer_slug, os.path.getctime(path), os.path.getmtime(path))
grid = None
if from_disk:
logger.debug(
'Memory from file...',
extra={'subgrid_id': self.subgrid_id})
layer_slug = kwargs['layers'].split(':')[0]
logger.debug(layer_slug)
grid_path = os.path.join(config.DATA_DIR, '3di', layer_slug,
'grids.nc')
if ('file-memory' in self.grid and
self.grid['file-memory'] == generate_hash(
grid_path, layer_slug)):
# already loaded
# if a new file is placed in the same location, it is not
# detected!!
logger.debug(
'already loaded from file into memory',
extra={'subgrid_id': self.subgrid_id})
grid = self.grid
else:
# load file into memory
logger.debug(
'loading file into memory',
extra={'subgrid_id': self.subgrid_id})
nc = Dataset(grid_path, 'r', format='NETCDF3_CLASSIC')
grid = {}
grid['dsnop'] = nc.variables['dsnop'].getValue()[0]
grid['wkt'] = ''.join(nc.variables['wkt'])
grid['dps'] = nc.variables['dps'][:].copy()
grid['maxdepth'] = nc.variables['maxdepth'][:].copy()
grid['arrival'] = nc.variables['arrival'][:].copy()
grid['x0p'] = nc.variables['x0p'].getValue()[0]
grid['y0p'] = nc.variables['y0p'].getValue()[0]
grid['x1p'] = nc.variables['x1p'].getValue()[0]
grid['y1p'] = nc.variables['y1p'].getValue()[0]
grid['dxp'] = nc.variables['dxp'].getValue()[0]
grid['dyp'] = nc.variables['dyp'].getValue()[0]
grid['file-memory'] = generate_hash(grid_path, layer_slug)
grid['layer-slug'] = layer_slug # needed for getcapabilities
self.grid = grid
nc.close()
if grid is None:
if not self.grid:
logger.info(
'Initializing grids (is normally already done, unless '
'some server error)',
extra={'subgrid_id': self.subgrid_id})
return None # Crashes, try again later!
grid = self.grid
# try to get parameters from request
srs = kwargs.get("srs")
bbox_str = kwargs.get("bbox")
if bbox_str:
bbox = [float(x) for x in bbox_str.split(",")]
else:
bbox = None
logger.debug('bbox: %r' % str(bbox))
height = int(kwargs.get("height", "0"))
width = int(kwargs.get("width", "0"))
# multiply the slicing stepsize with 'fast'.
fast = float(kwargs.get("fast", "1.4"))
if all([srs, bbox, height, width]):
logger.debug(
"slicing and dicing",
extra={'subgrid_id': self.subgrid_id})
src_srs = osgeo.osr.SpatialReference()
src_srs.ImportFromEPSGA(int(srs.split(':')[1]))
dst_srs = osgeo.osr.SpatialReference()
if 'wkt' in grid and grid['wkt']:
dst_srs.ImportFromWkt(grid["wkt"])
if dst_srs.GetAuthorityCode("PROJCS") == '28992' and \
not dst_srs.GetTOWGS84():
logger.error(
"Check WKT for TOWGS84 string! Je weet tog ;-)")
else:
logger.warning(
'Something is probably wrong with the wkt, taking '
'default 28992.',
extra={'subgrid_id': self.subgrid_id})
dst_srs.ImportFromEPSGA(28992)
src2dst = osgeo.osr.CoordinateTransformation(src_srs, dst_srs)
(xmin, ymin, xmax, ymax) = bbox
# Beware: the destination is NOT rectangular, so we need to
# recalculate the bbox.
x0_dst, y0_dst, _ = src2dst.TransformPoint(xmin, ymin)
x1_dst, y1_dst, _ = src2dst.TransformPoint(xmax, ymin)
x2_dst, y2_dst, _ = src2dst.TransformPoint(xmin, ymax)
x3_dst, y3_dst, _ = src2dst.TransformPoint(xmax, ymax)
xmin_dst = min([x0_dst, x1_dst, x2_dst, x3_dst])
xmax_dst = max([x0_dst, x1_dst, x2_dst, x3_dst])
ymin_dst = min([y0_dst, y1_dst, y2_dst, y3_dst])
ymax_dst = max([y0_dst, y1_dst, y2_dst, y3_dst])
# lookup required slice
xmin_src, ymin_src = (grid['x0p'], grid['y0p'])
xmax_src, ymax_src = (grid['x1p'], grid['y1p'])
dx_src, dy_src = (grid['dxp'], grid['dyp'])
x_src = np.arange(xmin_src, xmax_src, dx_src)
y_src = np.arange(ymin_src, ymax_src, dy_src)
# Lookup indices of plotted grid
# this can be done faster with a calculation
dps_shape = grid['dps'].shape
x_start = \
min(max(bisect.bisect(x_src, xmin_dst) - 1, 0), dps_shape[1]-1)
x_end = \
min(max(bisect.bisect(x_src, xmax_dst) + 1, 0), dps_shape[1])
y_start = \
min(max(bisect.bisect(y_src, ymin_dst) - 1, 0), dps_shape[0]-1)
y_end = \
min(max(bisect.bisect(y_src, ymax_dst) + 1, 0), dps_shape[0])
# lookup resolution: restricted to make it faster for big images
x_step = max(
trunc(fast * (x_end - x_start)) // min(width, 1200), 1)
y_step = max(
trunc(fast * (y_end - y_start)) // min(height, 800), 1)
num_pixels = \
(y_end - y_start) // y_step * (x_end - x_start) // x_step
logger.debug(
'Slice: y=%d,%d,%d x=%d,%d,%d width=%d height=%d, pixels=%d' %
(y_start, y_end, y_step, x_start, x_end, x_step, width, height,
num_pixels),
extra={'subgrid_id': self.subgrid_id})
S = np.s_[y_start:y_end:y_step, x_start:x_end:x_step]
# Compute transform for sliced grid
transform = (
grid["x0p"] + dx_src*x_start,
dx_src*x_step,
0,
grid["y0p"] + grid["dyp"]*y_start,
0,
grid["dyp"]*y_step
)
else:
logger.debug(
"couldn't find enough info in %s", kwargs,
extra={'subgrid_id': self.subgrid_id})
S = np.s_[:, :]
transform = self.transform
if layer == 'waterlevel' or layer == 'waterheight':
nodatavalue = 1e10
dps = grid['dps'][S].copy()
# Set the Deltares no data value.
dps[dps == self.grid['dsnop']] = nodatavalue
quad_grid = grid['quad_grid'][S]
mask = grid['quad_grid_dps_mask'][S]
s1 = self.grid['s1'].copy()
vol1 = self.grid['vol1']
if interpolate == 'nearest':
# useless option
waterheight = s1[quad_grid.filled(0)]
else:
# Here comes the 'Martijn interpolatie'.
L = self.L
if L is None:
logger.warn(
"Interpolation data not available",
extra={'subgrid_id': self.subgrid_id})
X, Y = self.X[S], self.Y[S]
try:
# Kaapstad gives IndexError
volmask = (vol1 == 0)[quad_grid]
L.values = np.ascontiguousarray(s1[:, np.newaxis])
waterheight = L(X, Y)
# now mask the waterlevels where we did not compute
# or where mask of the
mask = utils.logical_or_reduce(
[np.isnan(waterheight), mask, volmask])
waterheight = np.ma.masked_array(waterheight, mask=mask)
except IndexError:
# Fallback to nearest
# Kaapstad:
# IndexError: index 1085856568 is out of bounds for size
# 16473
logger.error(
'Interpolation crashed, falling back to nearest.',
extra={'subgrid_id': self.subgrid_id})
waterheight = s1[quad_grid.filled(0)]
# Log everything
exc_type, exc_value, exc_traceback = sys.exc_info()
for line in traceback.format_exception(
exc_type, exc_value, exc_traceback):
logger.debug(line)
if layer == 'waterlevel':
waterlevel = waterheight - (-dps)
# Gdal does not know about masked arrays, so we transform to
# an array with a nodatavalue
array = np.ma.masked_array(waterlevel, mask=mask).filled(
nodatavalue)
container = rasters.NumpyContainer(array, transform, self.wkt,
nodatavalue=nodatavalue)
elif layer == 'waterheight':
waterlevel = waterheight
# Strange: nodatavalue becomes 0, which is undesirable for
# getprofile
array = np.ma.masked_array(waterlevel, mask=mask).filled(-dps)
container = rasters.NumpyContainer(array, transform, self.wkt,
nodatavalue=nodatavalue)
return container
elif layer == 'dps':
dps = grid['dps'][S].copy()
# Set the Deltares no data value.
nodatavalue = 1e10
dps[dps == self.grid['dsnop']] = nodatavalue
container = rasters.NumpyContainer(
dps, transform, self.wkt, nodatavalue=nodatavalue)
return container
elif layer == 'uc':
quad_grid = grid['quad_grid'][S]
uc = grid['uc']
uc_norm = np.sqrt(np.sum(uc**2, axis=0))
assert uc_norm.shape[0] != 2, "wrong sum dimension"
container = rasters.NumpyContainer(
uc_norm[quad_grid], transform, self.wkt)
return container
elif layer == 'sg':
dps = grid['dps'][S].copy()
quad_grid = grid['quad_grid'][S]
sg = grid['sg']
groundwater_depth = -dps - sg[quad_grid]
# A trick to hold all depths inside model, 0's are filtered out.
groundwater_depth[np.ma.less_equal(groundwater_depth, 0.01)] = 0.01
# Set the Deltares no data value.
nodatavalue = 1e10
groundwater_depth[dps == self.grid['dsnop']] = nodatavalue
container = rasters.NumpyContainer(
groundwater_depth, transform, self.wkt,
nodatavalue=nodatavalue)
return container
elif layer == 'sg_abs':
dps = grid['dps'][S].copy()
quad_grid = grid['quad_grid'][S]
sg = grid['sg']
groundwater_level = sg[quad_grid]
# A trick to hold all depths inside model, 0's are filtered out.
# groundwater_depth[np.ma.less_equal(groundwater_depth,
# 0.01)] = 0.01
# Set the Deltares no data value.
nodatavalue = 1e10
groundwater_level[dps == self.grid['dsnop']] = nodatavalue
container = rasters.NumpyContainer(
groundwater_level, transform, self.wkt,
nodatavalue=nodatavalue)
return container
elif layer == 'quad_grid':
quad_grid = grid['quad_grid'][S]
container = rasters.NumpyContainer(
quad_grid, transform, self.wkt)
return container
elif layer == 'infiltration':
dps = grid['dps'][S].copy()
g = grid['infiltrationrate'][S].copy()
# Set the Deltares no data value.
nodatavalue = 1e10
g[dps == self.grid['dsnop']] = nodatavalue
container = rasters.NumpyContainer(
g, transform, self.wkt, nodatavalue=nodatavalue)
return container
elif layer == 'interception':
dps = grid['dps'][S].copy()
g = grid['maxinterception'][S].copy()
# Set the Deltares no data value.
nodatavalue = 1e10
g[dps == self.grid['dsnop']] = nodatavalue
container = rasters.NumpyContainer(
g, transform, self.wkt, nodatavalue=nodatavalue)
return container
elif layer == 'soil':
dps = grid['dps'][S].copy()
g = grid['soiltype'][S].copy()
# Set the Deltares no data value.
nodatavalue = 1e10
g[dps == self.grid['dsnop']] = nodatavalue
container = rasters.NumpyContainer(
g, transform, self.wkt, nodatavalue=nodatavalue)
return container
elif layer == 'crop':
dps = grid['dps'][S].copy()
g = grid['croptype'][S].copy()
# Set the Deltares no data value.
nodatavalue = 1e10
g[dps == self.grid['dsnop']] = nodatavalue
container = rasters.NumpyContainer(
g, transform, self.wkt, nodatavalue=nodatavalue)
return container
elif layer == 'maxdepth':
if not from_disk:
return None # does not work!
a = grid['maxdepth'][S].copy()
dps = grid['dps'][S].copy()
wkt = grid['wkt']
# Set the Deltares no data value.
nodatavalue = 1e10
a[dps == grid['dsnop']] = nodatavalue
# Strange stuff: no data value is not handled correctly in
# preprocessing
a[a > 10000] = nodatavalue
container = rasters.NumpyContainer(
a, transform, wkt, nodatavalue=nodatavalue)
return container
elif layer == 'arrival':
if not from_disk:
return None # does not work!
a = grid['arrival'][S].copy()
dps = grid['dps'][S].copy()
wkt = grid['wkt']
# Set the Deltares no data value.
nodatavalue = 1e10
a[dps == grid['dsnop']] = nodatavalue
container = rasters.NumpyContainer(
a, transform, wkt, nodatavalue=nodatavalue)
return container
else:
raise NotImplemented("working on it")
