def coamps_press_windxy_dataset(g_target, start, stop): """ Downloads COAMPS winds for the given period (see fetch_coamps_wind), trims to the bounds of g_target, and returns an xarray Dataset. """ fetch_coamps_wind(start, stop) xy_min = g_target.nodes['x'].min(axis=0) xy_max = g_target.nodes['x'].max(axis=0) pad = 10e3 crop = [xy_min[0] - pad, xy_max[0] + pad, xy_min[1] - pad, xy_max[1] + pad] dss = [] for recs in coamps_files(start, stop): timestamp = recs['wnd_utru']['timestamp'] timestamp_dt = utils.to_datetime(timestamp) timestamp_str = timestamp_dt.strftime('%Y-%m-%d %H:%M') # use the local file dirname to get the same model subdirectory # i.e. cencoos_4km cache_fn = os.path.join(os.path.dirname(recs['pres_msl']['local']), "%s.nc" % timestamp_dt.strftime('%Y%m%d%H%M')) if not os.path.exists(cache_fn): print(timestamp_str) # load the 3 fields: wnd_utru = field.GdalGrid(recs['wnd_utru']['local']) wnd_vtru = field.GdalGrid(recs['wnd_vtru']['local']) pres_msl = field.GdalGrid(recs['pres_msl']['local']) # Reproject to UTM: these come out as 3648m resolution, compared to 4km input. # Fine. 366 x 325. Crops down to 78x95 wnd_utru_utm = wnd_utru.warp("EPSG:26910").crop(crop) wnd_vtru_utm = wnd_vtru.warp("EPSG:26910").crop(crop) pres_msl_utm = pres_msl.warp("EPSG:26910").crop(crop) ds = xr.Dataset() ds['time'] = timestamp x, y = wnd_utru_utm.xy() ds['x'] = ('x', ), x ds['y'] = ('y', ), y # copy, in hopes that we can free up ram more quickly ds['wind_u'] = ('y', 'x'), wnd_utru_utm.F.copy() ds['wind_v'] = ('y', 'x'), wnd_vtru_utm.F.copy() ds['pres'] = ('y', 'x'), pres_msl_utm.F.copy() ds.to_netcdf(cache_fn) ds.close() ds = xr.open_dataset(cache_fn) ds.load() # force load of data ds.close() # and close out file handles dss.append(ds) # so this is all in ram. ds = xr.concat(dss, dim='time') return ds
def factory(attrs): geo_bounds = attrs['geom'].bounds if attrs['src_name'] == 'usgs_topobathy_2m': # A nice seamless 2m DEM from USGS. A bit blurred, so maybe more like 4m resolution, but # still quite nice. fn = opj( IDRIVE, 'BASELAYERS/Elevation_DerivedProducts/LiDAR 2005-2012 entire Bay Area from AECOM', 'USGS_TopoBathy/San_Francisco_TopoBathy_Elevation_2m.tif') return field.GdalGrid(fn, geo_bounds=geo_bounds) if attrs['src_name'] == 'usgs_topobathy_2m_remove_flat_ponds': # a postprocessing of a subset of the above, where broad areas of constant # elevation are removed. fn = '../sources/usgs_2m_remove_flat_ponds_v00.tif' return field.GdalGrid(fn, geo_bounds=geo_bounds) if attrs['src_name'] == 'merged_ponds_25m': #fn='../../sbsprp/SbayPondBathy2005/merged_ponds.tif' fn = '../sources/merged_ponds_2m_smoothed.tif' return field.GdalGrid(fn, geo_bounds=geo_bounds) if attrs['src_name'] == 'USGS Alviso 2010': # The 2010 data from USGS Open File Report 2011-1315, Amy Foxgrover et al Alviso data. fn = '../../usgs/bathymetry/alviso_ofr2011_1315/2010/2010_DEM_UTM_NAVD88.tif' return field.GdalGrid(fn, geo_bounds=geo_bounds) if attrs['src_name'] == 'NOAA Sidescan SBB02_1m': # Recent NOAA sidescan for subtidal portions of LSB fn = '../../noaa/bathy/sf_bay_sidescan/Area B -SSS Bathymetry/BAG/SBB02_1m.bag' dem = field.GdalGrid(fn, geo_bounds=geo_bounds) dem.F = dem.F[:, :, 0] # Drop the second channel return dem if attrs['src_name'] == 'interp_lines': fn = '../sources/interp_lines.tif' return field.GdalGrid(fn, geo_bounds=geo_bounds) if attrs['src_name'] == 'all_sloughs_061610': fn = '../sources/all_sloughs_061610.tif' return field.GdalGrid(fn, geo_bounds=geo_bounds) if attrs['src_name'] == 'alviso_sections': fn = '../sources/rma20170609/alviso_sections.tif' return field.GdalGrid(fn, geo_bounds=geo_bounds) if attrs['src_name'] == 'guadalupe_sections_remove_local_minima': fn = '../sources/guadalupe_sections_remove_local_minima.tif' return field.GdalGrid(fn, geo_bounds=geo_bounds) if attrs['src_name'].startswith('py:'): expr = attrs['src_name'][3:] # something like 'ConstantField(-1.0)' # a little sneaky... make it look like it's running # after a "from stompy.spatial.field import *" # and also it gets fields of the shapefile field_hash = dict(field.__dict__) field_hash['PointsInPoly'] = PointsInPoly # convert the attrs into a dict suitable for passing to eval attrs_dict = {} for name in attrs.dtype.names: attrs_dict[name] = attrs[name] return eval(expr, field_hash, attrs_dict) assert False
def factory(attrs): geo_bounds = attrs['geom'].bounds if attrs['src_name'].startswith('py:'): expr = attrs['src_name'][3:] # something like 'ConstantField(-1.0)' # a little sneaky... make it look like it's running # after a "from stompy.spatial.field import *" # and also it gets fields of the shapefile field_hash = dict(field.__dict__) # convert the attrs into a dict suitable for passing to eval attrs_dict = {} for name in attrs.dtype.names: attrs_dict[name] = attrs[name] return eval(expr, field_hash, attrs_dict) # Otherwise assume src_name is a file name or file pattern. for p in paths: full_path = os.path.join(p, attrs['src_name']) files = glob.glob(full_path) if len(files) > 1: mrf = field.MultiRasterField(files) return mrf elif len(files) == 1: gg = field.GdalGrid(files[0], geo_bounds=geo_bounds) gg.default_interpolation = 'linear' return gg log.warning("Source %s was not found -- ignoring" % attrs['src_name']) return None
def dem(suffix=''): # This DEM covers a larger area, but doesn't have the extra processing # around the junction base_fn = os.path.join(here, "junction-composite-dem.tif") if suffix == '': jct_fn = os.path.join(here, 'junction-composite-20190117-no_adcp.tif') elif suffix == 'smooth': jct_fn = os.path.join(here, 'junction-composite-20200603-w_smooth.tif') elif suffix == 'smoothadcp': jct_fn = os.path.join(here, 'junction-composite-20200604-w_smooth.tif') elif suffix == 'smoothadcp2': # this includes adcp-derived bathy above the junction, while # the above uses dwr multibeam above the junction jct_fn = os.path.join(here, 'junction-composite-20200605-w_smooth.tif') elif suffix == 'adcp': jct_fn = None else: raise Exception("Unknown bathy suffix/version: %s" % suffix) if jct_fn is not None: # Specify priority to be sure jct fn wins out f = field.MultiRasterField([(jct_fn, 10), (base_fn, 0)]) else: f = field.GdalGrid(base_fn) return f
def get_region_dem(self, bathy_fn): # And extract the "correct" answer from the DEM region_poly = self.hyp.g.boundary_polygon_by_union( np.nonzero(self.region_cells)[0]) self.region_poly = region_poly xyxy = region_poly.bounds region_dem = field.GdalGrid( bathy_fn, geo_bounds=[xyxy[0], xyxy[2], xyxy[1], xyxy[3]]) self.dem = region_dem dem_in_poly = region_dem.polygon_mask(region_poly) dem_bad = np.isnan(region_dem.F) if np.any(dem_bad & dem_in_poly): print("%d of %d dem pixels are missing" % ((dem_bad & dem_in_poly).sum(), dem_in_poly.sum())) dem_in_poly = dem_in_poly & (~dem_bad) pix_A = region_dem.dx * region_dem.dy pix_z = region_dem.F[dem_in_poly] self.pix_A = pix_A self.pix_z = pix_z return pix_A, pix_z
def factory(attrs): geo_bounds = attrs['geom'].bounds if attrs['src_name'].endswith('.tif'): for p in tif_paths: fn = opj(p, attrs['src_name']) if os.path.exists(fn): f = field.GdalGrid(fn, geo_bounds=geo_bounds) f.default_interpolation = 'linear' return f raise Exception("Could not find tif %s" % attrs['src_name']) elif attrs['src_name'].startswith('py:'): expr = attrs['src_name'][3:] # something like 'ConstantField(-1.0)' # a little sneaky... make it look like it's running # after a "from stompy.spatial.field import *" # and also it gets fields of the shapefile field_hash = dict(field.__dict__) # convert the attrs into a dict suitable for passing to eval attrs_dict = {} for name in attrs.dtype.names: attrs_dict[name] = attrs[name] return eval(expr, field_hash, attrs_dict) assert False
def factory(attrs): geo_bounds = attrs['geom'].bounds if attrs['src_name'] == 'usgs_2m_topobathy_steinberger': fns = [ 'steinberger-dem-crop.tif', ('/media/idrive/BASELAYERS/Elevation_DerivedProducts/' 'LiDAR 2005-2012 entire Bay Area from AECOM/USGS_TopoBathy/' 'San_Francisco_TopoBathy_Elevation_2m.tif') ] for fn in fns: if os.path.exists(fn): return field.GdalGrid(fn, geo_bounds=geo_bounds) else: raise Exception("Couldn't find the USGS 2m data") if attrs['src_name'].startswith('py:'): expr = attrs['src_name'][3:] # something like 'ConstantField(-1.0)' # a little sneaky... make it look like it's running # after a "from stompy.spatial.field import *" # and also it gets fields of the shapefile field_hash = dict(field.__dict__) # convert the attrs into a dict suitable for passing to eval attrs_dict = {} for name in attrs.dtype.names: attrs_dict[name] = attrs[name] return eval(expr, field_hash, attrs_dict) assert False
def aerial(): img = field.GdalGrid( os.path.join( here, "../../../gis/aerial/m_3712114_sw_10_h_20160621_20161004-UTM-crop.tif" )) # Alpha channel is mis-scaled. Drop it. img.F = img.F[..., :3] return img
def dem_and_levees(): dem_and_levees_fn = 'dem_and_levees.tif' if force or not os.path.exists(dem_and_levees_fn): assert np.allclose(dem.extents, levee_burn.extents) assert dem.F.shape == levee_burn.F.shape leveed_dem = dem.copy() leveed_dem.F = np.fmax(dem.F, levee_burn.F) os.path.exists(dem_and_levees_fn) and os.unlink(dem_and_levees_fn) leveed_dem.write_gdal(dem_and_levees_fn) else: leveed_dem = field.GdalGrid(dem_and_levees_fn) return leveed_dem
def dem_analysis(self, bathy_fn): # 1.0 is from half the current DEM resolution pad = 10.0 bounds = [ self.section_ls[:, 0].min() - pad, self.section_ls[:, 0].max() + pad, self.section_ls[:, 1].min() - pad, self.section_ls[:, 1].max() + pad ] self.dem = field.GdalGrid(bathy_fn, geo_bounds=bounds) res = 0.5 * self.dem.dx self.section_segs = linestring_utils.resample_linearring( self.section_ls, res, closed_ring=0) self.section_z = self.dem(self.section_segs) self.section_s = utils.dist_along(self.section_segs)
def factory(attrs): geo_bounds = attrs['geom'].bounds if attrs['src_name'] == 'dwr_2m_dems': mrf = field.MultiRasterField([os.path.join(dwr_dem_path, "*_2m*tif")]) return mrf elif attrs['src_name'] == 'dwr_10m_dems': #mrf=field.MultiRasterField([os.path.join(dwr_dem_path,"dem_bay_delta_10m_v3_20121109_*.tif")]) # In this case there is only one that matters -- tile 4. gg = field.GdalGrid(os.path.join( dwr_dem_path, "dem_bay_delta_10m_v3_20121109_4.tif"), geo_bounds=geo_bounds) gg.default_interpolation = 'linear' return gg elif attrs['src_name'].endswith('.tif'): for p in tif_paths: fn = opj(p, attrs['src_name']) if os.path.exists(fn): gg = field.GdalGrid(fn, geo_bounds=geo_bounds) gg.default_interpolation = 'linear' return gg raise Exception("Could not find tif %s" % attrs['src_name']) elif attrs['src_name'].startswith('py:'): expr = attrs['src_name'][3:] # something like 'ConstantField(-1.0)' # a little sneaky... make it look like it's running # after a "from stompy.spatial.field import *" # and also it gets fields of the shapefile field_hash = dict(field.__dict__) # convert the attrs into a dict suitable for passing to eval attrs_dict = {} for name in attrs.dtype.names: attrs_dict[name] = attrs[name] return eval(expr, field_hash, attrs_dict) assert False
def f(args): fn, xxyy, res = args # temporary code to set projection info: if os.path.exists(fn): dem = field.GdalGrid(fn) if dem.projection() == '': dem.assign_projection('EPSG:26910') os.unlink(fn) dem.write_gdal(fn) if not os.path.exists(fn): try: dem = mbf.to_grid(dx=res, dy=res, bounds=xxyy) # not great, since we're not padding the borders, but # there is very little filling now that the 2m dataset # is so pervasive. fill_holes(dem) dem.write_gdal(fn) except Exception as exc: print "Failed with exception" print repr(exc)
def lidar_resamp(): lidar_resamp_fn = 'lidar-levees.tif' if not os.path.exists(lidar_resamp_fn): # Will compile the lidar data here: lidar_levees = field.SimpleGrid(extents=lidar_mask().extents, F=np.zeros(lidar_mask().F.shape, np.float32)) lidar_levees.F[:, :] = np.nan lidar_resamp = lidar_src.to_grid(bounds=lidar_mask().extents, dx=lidar_mask().dx, dy=lidar_mask().dy) lidar_resamp.F[~lidar_mask().F] = np.nan # There are seams, at least 1px wide, between lidar tiles. lidar_resamp.fill_by_convolution(iterations=1) lidar_resamp.write_gdal('lidar-levees.tif') else: lidar_resamp = field.GdalGrid('lidar-levees.tif') return lidar_resamp
""" The in-pond data is very blocky - apply a bit of smoothing before it gets used in the dem """ import matplotlib.pyplot as plt from stompy.spatial import field ## fn = '../../sbsprp/SbayPondBathy2005/merged_ponds.tif' ponds_dem = field.GdalGrid(fn) # limit it to the parts we're actually using ponds_crop = ponds_dem.extract_tile(xxyy=(579160, 592550, 4141950, 4146580), res=2, interpolation='bilinear') # expand a bit ponds_crop.fill_by_convolution(iterations=5, kernel_size=5) # Smooth it ponds_crop.smooth_by_convolution(iterations=5) ## plt.figure(2).clf() fig, ax = plt.subplots(num=2) ponds_crop.plot(ax=ax, interpolation='nearest') ## # And looks like we need to convert to m, too. ponds_crop.F *= 0.3048
v04: look at 2018 data, first with an eye to detecting flows. """ import pandas as pd import xarray as xr from stompy import utils import numpy as np import pystan import matplotlib.pyplot as plt from stompy.spatial import field import seawater from stompy.plot import plot_utils ## img = field.GdalGrid("../../model/grid/boundaries/junction-bathy-rgb.tif") ## ds = xr.open_dataset('pings-2018-03-20T20:00_2018-03-20T22:00.nc') # SM4, SM3 ds = ds.isel(rx=[2, 3]) ## beacons = ds.rx_beacon.values beacon_to_xy = dict([(ds.rx_beacon.values[i], (ds.rx_x.values[i], ds.rx_y.values[i])) for i in range(ds.dims['rx'])]) ## # can get a little bit of constraint from 2-rx pings, but
upper_ll = [-120.930408, 37.309940] lower_ll = [-121.271098, 37.691794] release_xy = proj_utils.mapper('WGS84', 'EPSG:26910')([upper_ll, lower_ll]) ## dem_mrg = field.MultiRasterField([ "../../../bathy/junction-composite-20200605-w_smooth.tif", "/home/rusty/data/bathy_dwr/gtiff/dem_bay_delta_10m_v3_20121109_2.tif" ]) clip = (646390., 648336., 4184677., 4187210.) dem = dem_mrg.extract_tile(clip) ## aerial = field.GdalGrid( "../../../gis/aerial/m_3712114_sw_10_h_20160621_20161004-UTM.tif") aerial.F = aerial.F[:, :, :3] # drop alpha - seems mis-scaled ## #grid=unstructured_grid.UnstructuredGrid.read_ugrid("../../../model/grid/snubby_junction/snubby-06.nc") grid = unstructured_grid.UnstructuredGrid.read_ugrid( "../../../model/suntans/snubby-08-edit60-with_bathysmooth.nc") grid_poly = grid.boundary_polygon() ## rx_locs = pd.read_csv("../../../field/hor_yaps/yap-positions.csv") ##
# 03: fix initial condition # 04: conveyance2d=-1 # 05: bedlevtype=6 model.set_run_dir("runs/hypso_05", mode='askclobber') model.run_start = np.datetime64('2017-12-01') model.run_stop = np.datetime64('2017-12-10') model.load_mdu('template.mdu') src_grid = '../grid/CacheSloughComplex_v111-edit19.nc' dst_grid = os.path.basename(src_grid).replace('.nc', '-bathy.nc') bathy_fn = "../bathy/merged_2m-20181113.tif" if utils.is_stale(dst_grid, [src_grid, bathy_fn]): g = unstructured_grid.UnstructuredGrid.from_ugrid(src_grid) dem = field.GdalGrid(bathy_fn) node_depths = dem(g.nodes['x']) while np.any(np.isnan(node_depths)): missing = np.nonzero(np.isnan(node_depths))[0] print("Looping to fill in %d missing node depths" % len(missing)) for n in missing: nbrs = g.node_to_nodes(n) node_depths[n] = np.nanmean(node_depths[nbrs]) g.add_node_field('depth', node_depths, on_exists='overwrite') g.write_ugrid(dst_grid, overwrite=True) else: g = unstructured_grid.UnstructuredGrid.from_ugrid(dst_grid) model.set_grid(g) # I think this doesn't matter with conveyance2d>0 # 6 means cells=mean(nodes), edges=shallower(cells)
import os from stompy.grid import unstructured_grid from stompy.spatial import field from stompy import utils import matplotlib.pyplot as plt import numpy as np ## bathy_dir = "../../../bathy" bathy_existing_fn = os.path.join(bathy_dir, 'compiled-dem-existing-20210608-1m.tif') dem = field.GdalGrid(bathy_existing_fn) # grid_dir="../../../grid" # grid_fn=os.path.join(grid_dir,'quad_tri_v21-edit05.nc') # g=unstructured_grid.UnstructuredGrid.read_ugrid(grid_fn) # g.renumber() ## g = unstructured_grid.UnstructuredGrid.read_ugrid( 'pesca_butano_existing_deep_bathy.nc') ## alpha = np.linspace(0, 1, 5) edge_data = np.zeros((g.Nedges(), 3), np.float64)
import numpy as np import matplotlib.pyplot as plt from matplotlib import gridspec import track_common from statsmodels.graphics.tsaplots import plot_pacf, plot_acf from statsmodels.tsa import stattools from stompy import nanpsd from stompy import utils from stompy.spatial import field import seaborn as sns import stompy.plot.cmap as scmap turbo = scmap.load_gradient('turbo.cpt') ## dem = field.GdalGrid("../../bathy/junction-composite-dem-no_adcp.tif") clip = (647167.570005404, 647416.10024743, 4185000, 4186500) demcrop = dem.crop(clip) ## def unwrap(angles): deltas = (np.diff(angles) + np.pi) % (2 * np.pi) - np.pi return np.cumsum(np.r_[angles[0], deltas]) def expand(data, time, dt_nom=5.0): # fill, based on nominal tag interval of 5 seconds di = np.round(np.diff(time) / dt_nom).astype(np.int32) assert di.min() > 0
from shapely import geometry from scipy import ndimage from stompy.spatial import (field, wkb2shp, linestring_utils) from stompy import utils import stompy.plot.cmap as scm from stompy.grid import unstructured_grid from stompy.model.delft import io data_root = '/opt/data' ## dem = field.GdalGrid( os.path.join(data_root, 'bathy_interp/master2017/tiles_2m_20171024/merged_2m.tif')) ## # More details on the provenance of this on the Google drive in # Modeling/Lower South Bay/shoreline elevs inv_fn = "linear_features-v02.shp" inv = wkb2shp.shp2geom(inv_fn) ## # For the moment, focus on just the parts of the inventory which overlap with the DEM. bounds_xyxy = np.array(dem.extents)[[0, 2, 1, 3]] box = geometry.box(*bounds_xyxy)
def base_config(model): """ model: Model instance, should already have run_dir set. """ model.num_procs = 4 model.z_datum = 'NAVD88' model.projection = 'EPSG:26910' model.utc_offset = np.timedelta64(-8, 'h') # PST model.run_start = np.datetime64('2017-12-01') # very short! model.run_stop = np.datetime64('2017-12-05') model.load_mdu('template.mdu') src_grid = '../grid/CacheSloughComplex_v111-edit19fix.nc' dst_grid = os.path.basename(src_grid).replace('.nc', '-bathy.nc') bathy_fn = "../bathy/merged_2m-20181113.tif" if utils.is_stale(dst_grid, [src_grid, bathy_fn]): g = unstructured_grid.UnstructuredGrid.from_ugrid(src_grid) dem = field.GdalGrid(bathy_fn) # node_depths=dem(g.nodes['x']) import dem_cell_node_bathy node_depths = dem_cell_node_bathy.dem_to_cell_node_bathy(dem, g) g.add_node_field('depth', node_depths, on_exists='overwrite') g.write_ugrid(dst_grid, overwrite=True) else: g = unstructured_grid.UnstructuredGrid.from_ugrid(dst_grid) model.set_grid(g) # I think this doesn't matter with conveyance2d>0 model.mdu['geometry', 'BedlevType'] = 6 # ostensibly analytic 2D conveyance with 3. # 2 is faster, and appears less twitchy while showing very similar # calibration results. # 0 blocks some flow, 1 was a little twitchy. model.mdu['geometry', 'Conveyance2D'] = -1 # enabling this seems to cause a lot of oscillation. model.mdu['geometry', 'Nonlin2D'] = 0 model.mdu['output', 'MapInterval'] = 7200 #model.mdu['output','WaqInterval']=1800 model.mdu['physics', 'UnifFrictCoef'] = 0.025 model.set_cache_dir('cache') model.add_gazetteer('gis/model-features.shp') model.add_gazetteer('gis/point-features.shp') hm.SourceSinkBC.dredge_depth = -1 hm.FlowBC.dredge_depth = -1 # check_bspp.py has the code that converted original tim to csv. # awkward reloading of that, but at least it's independent of the # simulation period. model.add_bcs(barker_data.BarkerPumpsBC(name='Barker_Pumping_Plant')) # Decker only exists post-2015 if model.run_start > np.datetime64("2015-11-16"): model.add_bcs( hm.NwisStageBC(name='decker', station=11455478, cache_dir='cache')) else: # maybe fall back to Rio Vista, or some adjustment thereof raise Exception( "Decker tidal data starts 2015-11-16, too late for this simulation period" ) model.add_bcs( hm.NwisFlowBC(name='threemile', station=11337080, cache_dir='cache')) # GSS: from compare_flows and lit, must be flipped. model.add_bcs( hm.NwisFlowBC(name='Georgiana', station=11447903, cache_dir='cache', filters=[hm.Transform(lambda x: -x)])) model.add_bcs( hm.NwisFlowBC(name='dcc', station=11336600, cache_dir='cache', filters=[FillGaps(), hm.Transform(lambda x: -x)])) sac = hm.NwisFlowBC( name="SacramentoRiver", station=11447650, pad=np.timedelta64(5, 'D'), cache_dir='cache', filters=[hm.LowpassGodin(), hm.Lag(np.timedelta64(-2 * 3600, 's'))]) model.add_bcs(sac) ## pad = np.timedelta64(5, 'D') lisbon_ds = cdec.cdec_dataset(station='LIS', start_date=model.run_start - pad, end_date=model.run_stop + pad, sensor=20, cache_dir=cache_dir) # to m3/s lisbon_ds['Q'] = lisbon_ds['sensor0020'] * 0.028316847 # to hourly average lisbon_ds.groupby(lisbon_ds.time.astype('M8[h]')).mean() lisbon_bc = hm.FlowBC(name='lis', Q=lisbon_ds.Q) # Roughness - handled by caller. # model.add_RoughnessBC(shapefile='forcing-data/manning_n.shp') # Culvert at CCS if 1: # rough accounting of restoration if model.run_start < np.datetime64("2014-11-01"): # name => id # polylinefile is filled in from shapefile and name model.add_Structure( name='ccs_breach', type='gate', door_height=15, # no overtopping? lower_edge_level=0.89, opening_width=0.0, # pretty sure this is ignored. sill_level=0.85, # gives us a 0.05m opening? horizontal_opening_direction='symmetric') # these are really just closed for levee_name in ['ccs_west', 'ccs_east']: model.add_Structure(name=levee_name, type='gate', door_height=15, lower_edge_level=3.5, opening_width=0.0, sill_level=3.5, horizontal_opening_direction='symmetric') mon_sections = model.match_gazetteer(monitor=1, geom_type='LineString') mon_points = model.match_gazetteer(geom_type='Point') model.add_monitor_sections(mon_sections) model.add_monitor_points(mon_points) return model
def dem(): return field.GdalGrid( os.path.join( here, "../../../bathy/junction-composite-20200604-w_smooth.tif"))
def set_lsb_bathy(grid): check_bathy() # First, load in the original sfb_dfm grid to get bathymetry log.info("Loading SFB DFM v2 grid") sfb_dfm_grid=xr.open_dataset('sfb_dfm/sfei_v19_net.nc') sfb_X=np.c_[ sfb_dfm_grid.NetNode_x.values, sfb_dfm_grid.NetNode_y.values] sfb_dfm_field=field.XYZField(X=sfb_X, F=sfb_dfm_grid.NetNode_z.values) lsb_X=grid.nodes['x'] # This will set all points within the convex hull of the original # grid. These elevations are used in the output (a) outside the # LSB merged_2m DEM, and (b) to prioritize which nodes to use when # moving depths from edges to nodes. # Could argue that it would be better to pull point elevations here # from the DEM where they overlap. Probably makes very little difference lsb_z=sfb_dfm_field(lsb_X) # still some nans at this point # log.info("Loading LSB dem from %s"%merged_2m_path) dem=field.GdalGrid(merged_2m_path) # # Use the 2m DEM to find an effective minimum depth for each edge covered # by the DEM. edge_min_depths=depth_connectivity.edge_connection_depth(grid,dem,centers='lowest') # move those edge depths to node depths node_depths=depth_connectivity.greedy_edgemin_to_node(grid,lsb_z,edge_min_depths) # Still have some nodes with nan depth, first try filling in with the DEM. missing=np.isnan(node_depths) node_depths[missing]=dem( lsb_X[missing,:] ) # And wrap it up with a more forgiving interpolation from the original sfb_dfm_v2 # grid (about 12 points on the convex hull) still_missing=np.isnan(node_depths) node_depths[still_missing]=sfb_dfm_field.interpolate( lsb_X[still_missing,:], 'nearest' ) assert np.isnan(node_depths).sum()==0 # Update the grid grid.nodes['depth']=node_depths if 0: # caller is going to deal with I/O out_file='lsb_v99_bathy_net.nc' os.path.exists(out_file) and os.unlink(out_file) dfm_grid.write_dfm(grid,out_file) if 0: # plot for development. plt.figure(10).clf() fig,ax=plt.subplots(num=10) edge_mins=grid.nodes['depth'][grid.edges['nodes']].min(axis=1) ecoll=grid.plot_edges(lw=1.5,values=edge_mins) ncoll=grid.plot_nodes(values=grid.nodes['depth']) plt.setp([ecoll,ncoll],clim=[-3,3]) plot_utils.cbar(ncoll,extras=[ecoll]) # Modified in place, but return just in case return grid # QED.
def bathy(): from stompy.spatial import field return field.GdalGrid( '../bathy/OldRvr_at_SanJoaquinRvr2012_0104-utm-m_NAVD.tif')
if gen_grids: grid_dir = "../../../grid" grid_fn = os.path.join(grid_dir, 'quad_tri_v21-edit08.nc') g = unstructured_grid.UnstructuredGrid.read_ugrid(grid_fn) g.renumber() if gen_weirs: lines = wkb2shp.shp2geom('line_features.shp') for dem_fn, name in dems: if not os.path.exists(dem_fn): print("---- DEM file %s doesn't exist ----" % dem_fn) continue print("Loading DEM for %s" % name) dem = field.GdalGrid(dem_fn) if gen_weirs: print("Generating fixed weirs") fixed_weir_fn = "fixed_weirs-%s.pliz" % name fixed_weirs = [] # suitable for write_pli dx = 5.0 # m. discretize lines at this resolution for i in range(len(lines)): feat = lines[i] if feat['type'] != 'fixed_weir': continue print(f"Processing levee feature {feat['name']}") xy = np.array(feat['geom']) xy = linestring_utils.upsample_linearring(xy, dx, closed_ring=False)
def coamps_dataset(bounds, start, stop, cache_dir, fields=['wnd_utru', 'wnd_vtru', 'pres_msl'], missing='omit', fetch=True): """ Downloads COAMPS winds for the given period (see fetch_coamps_wind), trims to the bounds xxyy, and returns an xarray Dataset. fields: these are the coamps names for the fields. for legacy reasons, some of them are remapped in the dataset wnd_utru => wind_u wnd_vtru => wind_v pres_msl => pres missing:'omit' - if any of the fields are missing for a timestep, omit that timestep. None: will raise an error when GdalGrid fails. may in the future also have 'nan' which would fill that variable with nan. fetch: if False, rely on cached data. """ fields = list(fields) fields.sort() if fetch: fetch_coamps_wind(start, stop, cache_dir, fields=fields) pad = 10e3 crop = [bounds[0] - pad, bounds[1] + pad, bounds[2] - pad, bounds[3] + pad] dss = [] renames = {'wnd_utru': 'wind_u', 'wnd_vtru': 'wind_v', 'pres_msl': 'pres'} for recs in coamps_files(start, stop, cache_dir, fields=fields): timestamp = recs[fields[0]]['timestamp'] timestamp_dt = utils.to_datetime(timestamp) timestamp_str = timestamp_dt.strftime('%Y-%m-%d %H:%M') # use the local file dirname to get the same model subdirectory # i.e. cencoos_4km cache_fn = os.path.join( os.path.dirname(recs[fields[0]]['local']), "%s-%s.nc" % (timestamp_dt.strftime('%Y%m%d%H%M'), "-".join(fields))) if not os.path.exists(cache_fn): print(timestamp_str) ds = xr.Dataset() ds['time'] = timestamp # check to see that all files exists: # this will log the missing urls, which can then be added to the missing # list above if they are deemed really missing missing_urls = [ recs[field_name]['url'] for field_name in fields if not os.path.exists(recs[field_name]['local']) ] if len(missing_urls) > 0 and missing == 'omit': log.warning("Missing files for time %s: %s" % (timestamp_str, ", ".join(missing_urls))) continue for i, field_name in enumerate(fields): # load the grib file raw = field.GdalGrid(recs[field_name]['local']) # Reproject to UTM: these come out as 3648m resolution, compared to 4km input. # Fine. 366 x 325. Crops down to 78x95 raw_utm = raw.warp("EPSG:26910").crop(crop) if i == 0: # take spatial details from the first field x, y = raw_utm.xy() ds['x'] = ('x', ), x ds['y'] = ('y', ), y ds_name = renames.get(field_name, field_name) # copy, in hopes that we can free up ram more quickly ds[ds_name] = ('y', 'x'), raw_utm.F.copy() ds.to_netcdf(cache_fn) ds.close() ds = xr.open_dataset(cache_fn) ds.load() # force load of data ds.close() # and close out file handles dss.append(ds) # so this is all in ram. ds = xr.concat(dss, dim='time') return ds
date_str = "20201230" clip = zoom = (551800, 553290, 4124100, 4125400) # cbec surfaces: # For starters load just the N marsh and pond portion. # For compilation at the end load the whole thing if version == 'asbuilt': cbec_dem_fn = '../data/cbec/to_RCD/asbuilt/merged.tif' cbec_dem_name = "cbec As Built" elif version == 'existing': cbec_dem_fn = '../data/cbec/to_RCD/existing_grade/merged.tif' cbec_dem_name = "cbec Existing Grade" cbec_dem = field.GdalGrid(cbec_dem_fn, geo_bounds=clip) cbec_full_dem = field.GdalGrid(cbec_dem_fn) cbec_full_dem.name = cbec_dem.name = cbec_dem_name # as_built_extents,res = field.GdalGrid.metadata('../data/cbec/to_RCD/asbuilt/merged.tif') # existing_dem = field.GdalGrid('../data/cbec/to_RCD/existing_grade/merged.tif',geo_bounds=clip) # existing_dem.name="cbec Existing Grade" # 2017 CoNED LIDAR lidar2017 = field.GdalGrid( '../data/noaa/lidar/2017CoNED-north_marsh-lidar/Job558295_cent_ca_coned.tif', geo_bounds=clip) lidar2017.name = "CoNED 2017 LiDaR" lidar2011 = field.GdalGrid( '../data/noaa/lidar/2011CoastCons-north_marsh-lidar/Job558340_CA2009_coastal_DEM.tif', geo_bounds=clip)
from scipy import ndimage as ndi from stompy.spatial import field import numpy as np import matplotlib.pyplot as plt import os opj = os.path.join IDRIVE = '/media/idrive' ## fn = '../../sbsprp/SbayPondBathy2005/merged_ponds.tif' ponds_dem = field.GdalGrid(fn) zoom = ponds_dem.extents if 1: fn = opj( IDRIVE, 'BASELAYERS/Elevation_DerivedProducts/LiDAR 2005-2012 entire Bay Area from AECOM', 'USGS_TopoBathy/San_Francisco_TopoBathy_Elevation_2m.tif') geo_bounds = ponds_dem.extents # [550000,560000,4.14e6,4.14e6+10e3] dem = field.GdalGrid(fn, geo_bounds=geo_bounds) else: dem = field.GdalGrid('tiles_2m_20170508/merged_2m.tif', geo_bounds=zoom) ## plt.figure(1).clf() fig, ax = plt.subplots(num=1)
## bounds = wkb2shp.shp2geom('region-bounds-v00.shp') ## from shapely import ops one_poly = ops.cascaded_union(bounds['geom']) bounds_xyxy = one_poly.bounds ## dem = field.GdalGrid( ('/media/idrive/BASELAYERS/Elevation_DerivedProducts/' 'LiDAR 2005-2012 entire Bay Area from AECOM/USGS_TopoBathy/' 'San_Francisco_TopoBathy_Elevation_2m.tif'), geo_bounds=[ bounds_xyxy[0], bounds_xyxy[2], bounds_xyxy[1], bounds_xyxy[3] ]) ## plt.figure(1).clf() fig, ax = plt.subplots(num=1) # DEM is (3189, 3544) # Restricting to elevation below 2.155, still 8M pixels. img = dem.downsample(4).plot(vmin=-8, vmax=5.0, cmap='gray', interpolation='nearest', ax=ax)
coords=dict(time=[ model.run_start - 24 * h, model.run_start, model.run_start + ramp_hours * h, model.run_stop, model.run_stop + 24 * h ])) Q_downstream = drv.FlowBC(name='SJ_downstream', Q=Qdown) h_old_river = drv.StageBC(name='Old_River', z=2.2 - 0.65) model.add_bcs([Q_upstream, Q_downstream, h_old_river]) model.write() #-- if 0: # Add in variable roughness ec = model.grid.edges_center() z0_map = field.GdalGrid("../../bathy/composite-roughness.tif") z0 = z0_map(ec) z0[np.isnan(z0)] = float(model.config['z0B']) model.ic_ds['z0B'] = ('time', 'Ne'), z0[None, :] model.write_ic_ds() # model.partition() model.sun_verbose_flag = '-v' model.run_simulation() # with the lower friction, get a vertical courant number of 5.25 # due to a cell that's 1mm thick.