def update_grids_bbox(self, bbox): """Update quad_grid_dps_mask using bbox bbox format: [x0, x1, y0, y1] """ if 'quad_grid_dps_mask' not in self.grid: logger.debug( "Calling update_grids instead of update_grids_bbox.", extra={'subgrid_id': self.subgrid_id}) return self.update_grids() x0, x1, y0, y1 = bbox quad_grid = self.grid['quad_grid'][y0:y1, x0:x1] dps = self.grid['dps'][y0:y1, x0:x1] logger.debug( 'quad grid bbox shape: %r' % (str(quad_grid.shape)), extra={'subgrid_id': self.subgrid_id}) logger.debug( 'dps bbox shape: %r' % (str(dps.shape)), extra={'subgrid_id': self.subgrid_id}) # Sometimes quad_grid.mask is False instead of a table... (model Miami) # TODO: investigate more if quad_grid.mask.__class__.__name__ == 'bool_': mask = utils.logical_or_reduce([dps < -9000, ]) else: mask = utils.logical_or_reduce([quad_grid.mask, dps < -9000]) self.grid['quad_grid_dps_mask'][y0:y1, x0:x1] = mask
def update_grids(self): """Precalculate quad_grid_dps_mask that only needs to be calculated once. Needs to be run when quad_grid or dps is updated. """ grid = self.grid quad_grid = grid['quad_grid'] dps = grid['dps'] logger.debug( 'quad grid shape: %r' % (str(quad_grid.shape)), extra={'subgrid_id': self.subgrid_id}) logger.debug( 'dps shape: %r' % (str(dps.shape)), extra={'subgrid_id': self.subgrid_id}) # Sometimes quad_grid.mask is False instead of a table... (model Miami) # TODO: investigate more if quad_grid.mask.__class__.__name__ == 'bool_': mask = utils.logical_or_reduce([dps < -9000, ]) else: # 4 seconds mask = utils.logical_or_reduce([quad_grid.mask, dps < -9000]) if 'quad_grid_dps_mask' in grid: del self.grid['quad_grid_dps_mask'] self.grid['quad_grid_dps_mask'] = mask
def get(self, layer, interpolate='nearest', from_disk=False, **kwargs): """ layer: choose from waterlevel, waterheight, dps, uc, sg, quad_grid, infiltration, interception, soil, crop, maxdepth, arrival from_disk: read grids.nc instead of memory, kwargs must contain kw 'layers': duifp-duifp:maxdepth NOTE: maxdepth and arrival REQUIRE the from_disk method. """ def generate_hash(path, layer_slug): return '%r-%r-%r' % ( layer_slug, os.path.getctime(path), os.path.getmtime(path)) grid = None if from_disk: logger.debug( 'Memory from file...', extra={'subgrid_id': self.subgrid_id}) layer_slug = kwargs['layers'].split(':')[0] logger.debug(layer_slug) grid_path = os.path.join(config.DATA_DIR, '3di', layer_slug, 'grids.nc') if ('file-memory' in self.grid and self.grid['file-memory'] == generate_hash( grid_path, layer_slug)): # already loaded # if a new file is placed in the same location, it is not # detected!! logger.debug( 'already loaded from file into memory', extra={'subgrid_id': self.subgrid_id}) grid = self.grid else: # load file into memory logger.debug( 'loading file into memory', extra={'subgrid_id': self.subgrid_id}) nc = Dataset(grid_path, 'r', format='NETCDF3_CLASSIC') grid = {} grid['dsnop'] = nc.variables['dsnop'].getValue()[0] grid['wkt'] = ''.join(nc.variables['wkt']) grid['dps'] = nc.variables['dps'][:].copy() grid['maxdepth'] = nc.variables['maxdepth'][:].copy() grid['arrival'] = nc.variables['arrival'][:].copy() grid['x0p'] = nc.variables['x0p'].getValue()[0] grid['y0p'] = nc.variables['y0p'].getValue()[0] grid['x1p'] = nc.variables['x1p'].getValue()[0] grid['y1p'] = nc.variables['y1p'].getValue()[0] grid['dxp'] = nc.variables['dxp'].getValue()[0] grid['dyp'] = nc.variables['dyp'].getValue()[0] grid['file-memory'] = generate_hash(grid_path, layer_slug) grid['layer-slug'] = layer_slug # needed for getcapabilities self.grid = grid nc.close() if grid is None: if not self.grid: logger.info( 'Initializing grids (is normally already done, unless ' 'some server error)', extra={'subgrid_id': self.subgrid_id}) return None # Crashes, try again later! grid = self.grid # try to get parameters from request srs = kwargs.get("srs") bbox_str = kwargs.get("bbox") if bbox_str: bbox = [float(x) for x in bbox_str.split(",")] else: bbox = None logger.debug('bbox: %r' % str(bbox)) height = int(kwargs.get("height", "0")) width = int(kwargs.get("width", "0")) # multiply the slicing stepsize with 'fast'. fast = float(kwargs.get("fast", "1.4")) if all([srs, bbox, height, width]): logger.debug( "slicing and dicing", extra={'subgrid_id': self.subgrid_id}) src_srs = osgeo.osr.SpatialReference() src_srs.ImportFromEPSGA(int(srs.split(':')[1])) dst_srs = osgeo.osr.SpatialReference() if 'wkt' in grid and grid['wkt']: dst_srs.ImportFromWkt(grid["wkt"]) if dst_srs.GetAuthorityCode("PROJCS") == '28992' and \ not dst_srs.GetTOWGS84(): logger.error( "Check WKT for TOWGS84 string! Je weet tog ;-)") else: logger.warning( 'Something is probably wrong with the wkt, taking ' 'default 28992.', extra={'subgrid_id': self.subgrid_id}) dst_srs.ImportFromEPSGA(28992) src2dst = osgeo.osr.CoordinateTransformation(src_srs, dst_srs) (xmin, ymin, xmax, ymax) = bbox # Beware: the destination is NOT rectangular, so we need to # recalculate the bbox. x0_dst, y0_dst, _ = src2dst.TransformPoint(xmin, ymin) x1_dst, y1_dst, _ = src2dst.TransformPoint(xmax, ymin) x2_dst, y2_dst, _ = src2dst.TransformPoint(xmin, ymax) x3_dst, y3_dst, _ = src2dst.TransformPoint(xmax, ymax) xmin_dst = min([x0_dst, x1_dst, x2_dst, x3_dst]) xmax_dst = max([x0_dst, x1_dst, x2_dst, x3_dst]) ymin_dst = min([y0_dst, y1_dst, y2_dst, y3_dst]) ymax_dst = max([y0_dst, y1_dst, y2_dst, y3_dst]) # lookup required slice xmin_src, ymin_src = (grid['x0p'], grid['y0p']) xmax_src, ymax_src = (grid['x1p'], grid['y1p']) dx_src, dy_src = (grid['dxp'], grid['dyp']) x_src = np.arange(xmin_src, xmax_src, dx_src) y_src = np.arange(ymin_src, ymax_src, dy_src) # Lookup indices of plotted grid # this can be done faster with a calculation dps_shape = grid['dps'].shape x_start = \ min(max(bisect.bisect(x_src, xmin_dst) - 1, 0), dps_shape[1]-1) x_end = \ min(max(bisect.bisect(x_src, xmax_dst) + 1, 0), dps_shape[1]) y_start = \ min(max(bisect.bisect(y_src, ymin_dst) - 1, 0), dps_shape[0]-1) y_end = \ min(max(bisect.bisect(y_src, ymax_dst) + 1, 0), dps_shape[0]) # lookup resolution: restricted to make it faster for big images x_step = max( trunc(fast * (x_end - x_start)) // min(width, 1200), 1) y_step = max( trunc(fast * (y_end - y_start)) // min(height, 800), 1) num_pixels = \ (y_end - y_start) // y_step * (x_end - x_start) // x_step logger.debug( 'Slice: y=%d,%d,%d x=%d,%d,%d width=%d height=%d, pixels=%d' % (y_start, y_end, y_step, x_start, x_end, x_step, width, height, num_pixels), extra={'subgrid_id': self.subgrid_id}) S = np.s_[y_start:y_end:y_step, x_start:x_end:x_step] # Compute transform for sliced grid transform = ( grid["x0p"] + dx_src*x_start, dx_src*x_step, 0, grid["y0p"] + grid["dyp"]*y_start, 0, grid["dyp"]*y_step ) else: logger.debug( "couldn't find enough info in %s", kwargs, extra={'subgrid_id': self.subgrid_id}) S = np.s_[:, :] transform = self.transform if layer == 'waterlevel' or layer == 'waterheight': nodatavalue = 1e10 dps = grid['dps'][S].copy() # Set the Deltares no data value. dps[dps == self.grid['dsnop']] = nodatavalue quad_grid = grid['quad_grid'][S] mask = grid['quad_grid_dps_mask'][S] s1 = self.grid['s1'].copy() vol1 = self.grid['vol1'] if interpolate == 'nearest': # useless option waterheight = s1[quad_grid.filled(0)] else: # Here comes the 'Martijn interpolatie'. L = self.L if L is None: logger.warn( "Interpolation data not available", extra={'subgrid_id': self.subgrid_id}) X, Y = self.X[S], self.Y[S] try: # Kaapstad gives IndexError volmask = (vol1 == 0)[quad_grid] L.values = np.ascontiguousarray(s1[:, np.newaxis]) waterheight = L(X, Y) # now mask the waterlevels where we did not compute # or where mask of the mask = utils.logical_or_reduce( [np.isnan(waterheight), mask, volmask]) waterheight = np.ma.masked_array(waterheight, mask=mask) except IndexError: # Fallback to nearest # Kaapstad: # IndexError: index 1085856568 is out of bounds for size # 16473 logger.error( 'Interpolation crashed, falling back to nearest.', extra={'subgrid_id': self.subgrid_id}) waterheight = s1[quad_grid.filled(0)] # Log everything exc_type, exc_value, exc_traceback = sys.exc_info() for line in traceback.format_exception( exc_type, exc_value, exc_traceback): logger.debug(line) if layer == 'waterlevel': waterlevel = waterheight - (-dps) # Gdal does not know about masked arrays, so we transform to # an array with a nodatavalue array = np.ma.masked_array(waterlevel, mask=mask).filled( nodatavalue) container = rasters.NumpyContainer(array, transform, self.wkt, nodatavalue=nodatavalue) elif layer == 'waterheight': waterlevel = waterheight # Strange: nodatavalue becomes 0, which is undesirable for # getprofile array = np.ma.masked_array(waterlevel, mask=mask).filled(-dps) container = rasters.NumpyContainer(array, transform, self.wkt, nodatavalue=nodatavalue) return container elif layer == 'dps': dps = grid['dps'][S].copy() # Set the Deltares no data value. nodatavalue = 1e10 dps[dps == self.grid['dsnop']] = nodatavalue container = rasters.NumpyContainer( dps, transform, self.wkt, nodatavalue=nodatavalue) return container elif layer == 'uc': quad_grid = grid['quad_grid'][S] uc = grid['uc'] uc_norm = np.sqrt(np.sum(uc**2, axis=0)) assert uc_norm.shape[0] != 2, "wrong sum dimension" container = rasters.NumpyContainer( uc_norm[quad_grid], transform, self.wkt) return container elif layer == 'sg': dps = grid['dps'][S].copy() quad_grid = grid['quad_grid'][S] sg = grid['sg'] groundwater_depth = -dps - sg[quad_grid] # A trick to hold all depths inside model, 0's are filtered out. groundwater_depth[np.ma.less_equal(groundwater_depth, 0.01)] = 0.01 # Set the Deltares no data value. nodatavalue = 1e10 groundwater_depth[dps == self.grid['dsnop']] = nodatavalue container = rasters.NumpyContainer( groundwater_depth, transform, self.wkt, nodatavalue=nodatavalue) return container elif layer == 'sg_abs': dps = grid['dps'][S].copy() quad_grid = grid['quad_grid'][S] sg = grid['sg'] groundwater_level = sg[quad_grid] # A trick to hold all depths inside model, 0's are filtered out. # groundwater_depth[np.ma.less_equal(groundwater_depth, # 0.01)] = 0.01 # Set the Deltares no data value. nodatavalue = 1e10 groundwater_level[dps == self.grid['dsnop']] = nodatavalue container = rasters.NumpyContainer( groundwater_level, transform, self.wkt, nodatavalue=nodatavalue) return container elif layer == 'quad_grid': quad_grid = grid['quad_grid'][S] container = rasters.NumpyContainer( quad_grid, transform, self.wkt) return container elif layer == 'infiltration': dps = grid['dps'][S].copy() g = grid['infiltrationrate'][S].copy() # Set the Deltares no data value. nodatavalue = 1e10 g[dps == self.grid['dsnop']] = nodatavalue container = rasters.NumpyContainer( g, transform, self.wkt, nodatavalue=nodatavalue) return container elif layer == 'interception': dps = grid['dps'][S].copy() g = grid['maxinterception'][S].copy() # Set the Deltares no data value. nodatavalue = 1e10 g[dps == self.grid['dsnop']] = nodatavalue container = rasters.NumpyContainer( g, transform, self.wkt, nodatavalue=nodatavalue) return container elif layer == 'soil': dps = grid['dps'][S].copy() g = grid['soiltype'][S].copy() # Set the Deltares no data value. nodatavalue = 1e10 g[dps == self.grid['dsnop']] = nodatavalue container = rasters.NumpyContainer( g, transform, self.wkt, nodatavalue=nodatavalue) return container elif layer == 'crop': dps = grid['dps'][S].copy() g = grid['croptype'][S].copy() # Set the Deltares no data value. nodatavalue = 1e10 g[dps == self.grid['dsnop']] = nodatavalue container = rasters.NumpyContainer( g, transform, self.wkt, nodatavalue=nodatavalue) return container elif layer == 'maxdepth': if not from_disk: return None # does not work! a = grid['maxdepth'][S].copy() dps = grid['dps'][S].copy() wkt = grid['wkt'] # Set the Deltares no data value. nodatavalue = 1e10 a[dps == grid['dsnop']] = nodatavalue # Strange stuff: no data value is not handled correctly in # preprocessing a[a > 10000] = nodatavalue container = rasters.NumpyContainer( a, transform, wkt, nodatavalue=nodatavalue) return container elif layer == 'arrival': if not from_disk: return None # does not work! a = grid['arrival'][S].copy() dps = grid['dps'][S].copy() wkt = grid['wkt'] # Set the Deltares no data value. nodatavalue = 1e10 a[dps == grid['dsnop']] = nodatavalue container = rasters.NumpyContainer( a, transform, wkt, nodatavalue=nodatavalue) return container else: raise NotImplemented("working on it")