def coamps_press_windxy_dataset(g_target, start, stop):
"""
Downloads COAMPS winds for the given period (see fetch_coamps_wind),
trims to the bounds of g_target, and returns an xarray Dataset.
"""
fetch_coamps_wind(start, stop)
xy_min = g_target.nodes['x'].min(axis=0)
xy_max = g_target.nodes['x'].max(axis=0)
pad = 10e3
crop = [xy_min[0] - pad, xy_max[0] + pad, xy_min[1] - pad, xy_max[1] + pad]
dss = []
for recs in coamps_files(start, stop):
timestamp = recs['wnd_utru']['timestamp']
timestamp_dt = utils.to_datetime(timestamp)
timestamp_str = timestamp_dt.strftime('%Y-%m-%d %H:%M')
# use the local file dirname to get the same model subdirectory
# i.e. cencoos_4km
cache_fn = os.path.join(os.path.dirname(recs['pres_msl']['local']),
"%s.nc" % timestamp_dt.strftime('%Y%m%d%H%M'))
if not os.path.exists(cache_fn):
print(timestamp_str)
# load the 3 fields:
wnd_utru = field.GdalGrid(recs['wnd_utru']['local'])
wnd_vtru = field.GdalGrid(recs['wnd_vtru']['local'])
pres_msl = field.GdalGrid(recs['pres_msl']['local'])
# Reproject to UTM: these come out as 3648m resolution, compared to 4km input.
# Fine. 366 x 325. Crops down to 78x95
wnd_utru_utm = wnd_utru.warp("EPSG:26910").crop(crop)
wnd_vtru_utm = wnd_vtru.warp("EPSG:26910").crop(crop)
pres_msl_utm = pres_msl.warp("EPSG:26910").crop(crop)
ds = xr.Dataset()
ds['time'] = timestamp
x, y = wnd_utru_utm.xy()
ds['x'] = ('x', ), x
ds['y'] = ('y', ), y
# copy, in hopes that we can free up ram more quickly
ds['wind_u'] = ('y', 'x'), wnd_utru_utm.F.copy()
ds['wind_v'] = ('y', 'x'), wnd_vtru_utm.F.copy()
ds['pres'] = ('y', 'x'), pres_msl_utm.F.copy()
ds.to_netcdf(cache_fn)
ds.close()
ds = xr.open_dataset(cache_fn)
ds.load() # force load of data
ds.close() # and close out file handles
dss.append(ds) # so this is all in ram.
ds = xr.concat(dss, dim='time')
return ds
def factory(attrs):
geo_bounds = attrs['geom'].bounds
if attrs['src_name'] == 'usgs_topobathy_2m':
# A nice seamless 2m DEM from USGS. A bit blurred, so maybe more like 4m resolution, but
# still quite nice.
fn = opj(
IDRIVE,
'BASELAYERS/Elevation_DerivedProducts/LiDAR 2005-2012 entire Bay Area from AECOM',
'USGS_TopoBathy/San_Francisco_TopoBathy_Elevation_2m.tif')
return field.GdalGrid(fn, geo_bounds=geo_bounds)
if attrs['src_name'] == 'usgs_topobathy_2m_remove_flat_ponds':
# a postprocessing of a subset of the above, where broad areas of constant
# elevation are removed.
fn = '../sources/usgs_2m_remove_flat_ponds_v00.tif'
return field.GdalGrid(fn, geo_bounds=geo_bounds)
if attrs['src_name'] == 'merged_ponds_25m':
#fn='../../sbsprp/SbayPondBathy2005/merged_ponds.tif'
fn = '../sources/merged_ponds_2m_smoothed.tif'
return field.GdalGrid(fn, geo_bounds=geo_bounds)
if attrs['src_name'] == 'USGS Alviso 2010':
# The 2010 data from USGS Open File Report 2011-1315, Amy Foxgrover et al Alviso data.
fn = '../../usgs/bathymetry/alviso_ofr2011_1315/2010/2010_DEM_UTM_NAVD88.tif'
return field.GdalGrid(fn, geo_bounds=geo_bounds)
if attrs['src_name'] == 'NOAA Sidescan SBB02_1m':
# Recent NOAA sidescan for subtidal portions of LSB
fn = '../../noaa/bathy/sf_bay_sidescan/Area B -SSS Bathymetry/BAG/SBB02_1m.bag'
dem = field.GdalGrid(fn, geo_bounds=geo_bounds)
dem.F = dem.F[:, :, 0] # Drop the second channel
return dem
if attrs['src_name'] == 'interp_lines':
fn = '../sources/interp_lines.tif'
return field.GdalGrid(fn, geo_bounds=geo_bounds)
if attrs['src_name'] == 'all_sloughs_061610':
fn = '../sources/all_sloughs_061610.tif'
return field.GdalGrid(fn, geo_bounds=geo_bounds)
if attrs['src_name'] == 'alviso_sections':
fn = '../sources/rma20170609/alviso_sections.tif'
return field.GdalGrid(fn, geo_bounds=geo_bounds)
if attrs['src_name'] == 'guadalupe_sections_remove_local_minima':
fn = '../sources/guadalupe_sections_remove_local_minima.tif'
return field.GdalGrid(fn, geo_bounds=geo_bounds)
if attrs['src_name'].startswith('py:'):
expr = attrs['src_name'][3:]
# something like 'ConstantField(-1.0)'
# a little sneaky... make it look like it's running
# after a "from stompy.spatial.field import *"
# and also it gets fields of the shapefile
field_hash = dict(field.__dict__)
field_hash['PointsInPoly'] = PointsInPoly
# convert the attrs into a dict suitable for passing to eval
attrs_dict = {}
for name in attrs.dtype.names:
attrs_dict[name] = attrs[name]
return eval(expr, field_hash, attrs_dict)
assert False
def factory(attrs):
geo_bounds = attrs['geom'].bounds
if attrs['src_name'].startswith('py:'):
expr = attrs['src_name'][3:]
# something like 'ConstantField(-1.0)'
# a little sneaky... make it look like it's running
# after a "from stompy.spatial.field import *"
# and also it gets fields of the shapefile
field_hash = dict(field.__dict__)
# convert the attrs into a dict suitable for passing to eval
attrs_dict = {}
for name in attrs.dtype.names:
attrs_dict[name] = attrs[name]
return eval(expr, field_hash, attrs_dict)
# Otherwise assume src_name is a file name or file pattern.
for p in paths:
full_path = os.path.join(p, attrs['src_name'])
files = glob.glob(full_path)
if len(files) > 1:
mrf = field.MultiRasterField(files)
return mrf
elif len(files) == 1:
gg = field.GdalGrid(files[0], geo_bounds=geo_bounds)
gg.default_interpolation = 'linear'
return gg
log.warning("Source %s was not found -- ignoring" % attrs['src_name'])
return None
def dem(suffix=''):
# This DEM covers a larger area, but doesn't have the extra processing
# around the junction
base_fn = os.path.join(here, "junction-composite-dem.tif")
if suffix == '':
jct_fn = os.path.join(here, 'junction-composite-20190117-no_adcp.tif')
elif suffix == 'smooth':
jct_fn = os.path.join(here, 'junction-composite-20200603-w_smooth.tif')
elif suffix == 'smoothadcp':
jct_fn = os.path.join(here, 'junction-composite-20200604-w_smooth.tif')
elif suffix == 'smoothadcp2':
# this includes adcp-derived bathy above the junction, while
# the above uses dwr multibeam above the junction
jct_fn = os.path.join(here, 'junction-composite-20200605-w_smooth.tif')
elif suffix == 'adcp':
jct_fn = None
else:
raise Exception("Unknown bathy suffix/version: %s" % suffix)
if jct_fn is not None:
# Specify priority to be sure jct fn wins out
f = field.MultiRasterField([(jct_fn, 10), (base_fn, 0)])
else:
f = field.GdalGrid(base_fn)
return f
def get_region_dem(self, bathy_fn):
# And extract the "correct" answer from the DEM
region_poly = self.hyp.g.boundary_polygon_by_union(
np.nonzero(self.region_cells)[0])
self.region_poly = region_poly
xyxy = region_poly.bounds
region_dem = field.GdalGrid(
bathy_fn, geo_bounds=[xyxy[0], xyxy[2], xyxy[1], xyxy[3]])
self.dem = region_dem
dem_in_poly = region_dem.polygon_mask(region_poly)
dem_bad = np.isnan(region_dem.F)
if np.any(dem_bad & dem_in_poly):
print("%d of %d dem pixels are missing" %
((dem_bad & dem_in_poly).sum(), dem_in_poly.sum()))
dem_in_poly = dem_in_poly & (~dem_bad)
pix_A = region_dem.dx * region_dem.dy
pix_z = region_dem.F[dem_in_poly]
self.pix_A = pix_A
self.pix_z = pix_z
return pix_A, pix_z
def factory(attrs):
geo_bounds = attrs['geom'].bounds
if attrs['src_name'].endswith('.tif'):
for p in tif_paths:
fn = opj(p, attrs['src_name'])
if os.path.exists(fn):
f = field.GdalGrid(fn, geo_bounds=geo_bounds)
f.default_interpolation = 'linear'
return f
raise Exception("Could not find tif %s" % attrs['src_name'])
elif attrs['src_name'].startswith('py:'):
expr = attrs['src_name'][3:]
# something like 'ConstantField(-1.0)'
# a little sneaky... make it look like it's running
# after a "from stompy.spatial.field import *"
# and also it gets fields of the shapefile
field_hash = dict(field.__dict__)
# convert the attrs into a dict suitable for passing to eval
attrs_dict = {}
for name in attrs.dtype.names:
attrs_dict[name] = attrs[name]
return eval(expr, field_hash, attrs_dict)
assert False
def factory(attrs):
geo_bounds = attrs['geom'].bounds
if attrs['src_name'] == 'usgs_2m_topobathy_steinberger':
fns = [
'steinberger-dem-crop.tif',
('/media/idrive/BASELAYERS/Elevation_DerivedProducts/'
'LiDAR 2005-2012 entire Bay Area from AECOM/USGS_TopoBathy/'
'San_Francisco_TopoBathy_Elevation_2m.tif')
]
for fn in fns:
if os.path.exists(fn):
return field.GdalGrid(fn, geo_bounds=geo_bounds)
else:
raise Exception("Couldn't find the USGS 2m data")
if attrs['src_name'].startswith('py:'):
expr = attrs['src_name'][3:]
# something like 'ConstantField(-1.0)'
# a little sneaky... make it look like it's running
# after a "from stompy.spatial.field import *"
# and also it gets fields of the shapefile
field_hash = dict(field.__dict__)
# convert the attrs into a dict suitable for passing to eval
attrs_dict = {}
for name in attrs.dtype.names:
attrs_dict[name] = attrs[name]
return eval(expr, field_hash, attrs_dict)
assert False
def aerial():
img = field.GdalGrid(
os.path.join(
here,
"../../../gis/aerial/m_3712114_sw_10_h_20160621_20161004-UTM-crop.tif"
))
# Alpha channel is mis-scaled. Drop it.
img.F = img.F[..., :3]
return img
def dem_and_levees():
dem_and_levees_fn = 'dem_and_levees.tif'
if force or not os.path.exists(dem_and_levees_fn):
assert np.allclose(dem.extents, levee_burn.extents)
assert dem.F.shape == levee_burn.F.shape
leveed_dem = dem.copy()
leveed_dem.F = np.fmax(dem.F, levee_burn.F)
os.path.exists(dem_and_levees_fn) and os.unlink(dem_and_levees_fn)
leveed_dem.write_gdal(dem_and_levees_fn)
else:
leveed_dem = field.GdalGrid(dem_and_levees_fn)
return leveed_dem
def dem_analysis(self, bathy_fn):
# 1.0 is from half the current DEM resolution
pad = 10.0
bounds = [
self.section_ls[:, 0].min() - pad,
self.section_ls[:, 0].max() + pad,
self.section_ls[:, 1].min() - pad,
self.section_ls[:, 1].max() + pad
]
self.dem = field.GdalGrid(bathy_fn, geo_bounds=bounds)
res = 0.5 * self.dem.dx
self.section_segs = linestring_utils.resample_linearring(
self.section_ls, res, closed_ring=0)
self.section_z = self.dem(self.section_segs)
self.section_s = utils.dist_along(self.section_segs)
def factory(attrs):
geo_bounds = attrs['geom'].bounds
if attrs['src_name'] == 'dwr_2m_dems':
mrf = field.MultiRasterField([os.path.join(dwr_dem_path, "*_2m*tif")])
return mrf
elif attrs['src_name'] == 'dwr_10m_dems':
#mrf=field.MultiRasterField([os.path.join(dwr_dem_path,"dem_bay_delta_10m_v3_20121109_*.tif")])
# In this case there is only one that matters -- tile 4.
gg = field.GdalGrid(os.path.join(
dwr_dem_path, "dem_bay_delta_10m_v3_20121109_4.tif"),
geo_bounds=geo_bounds)
gg.default_interpolation = 'linear'
return gg
elif attrs['src_name'].endswith('.tif'):
for p in tif_paths:
fn = opj(p, attrs['src_name'])
if os.path.exists(fn):
gg = field.GdalGrid(fn, geo_bounds=geo_bounds)
gg.default_interpolation = 'linear'
return gg
raise Exception("Could not find tif %s" % attrs['src_name'])
elif attrs['src_name'].startswith('py:'):
expr = attrs['src_name'][3:]
# something like 'ConstantField(-1.0)'
# a little sneaky... make it look like it's running
# after a "from stompy.spatial.field import *"
# and also it gets fields of the shapefile
field_hash = dict(field.__dict__)
# convert the attrs into a dict suitable for passing to eval
attrs_dict = {}
for name in attrs.dtype.names:
attrs_dict[name] = attrs[name]
return eval(expr, field_hash, attrs_dict)
assert False
def f(args):
fn, xxyy, res = args
# temporary code to set projection info:
if os.path.exists(fn):
dem = field.GdalGrid(fn)
if dem.projection() == '':
dem.assign_projection('EPSG:26910')
os.unlink(fn)
dem.write_gdal(fn)
if not os.path.exists(fn):
try:
dem = mbf.to_grid(dx=res, dy=res, bounds=xxyy)
# not great, since we're not padding the borders, but
# there is very little filling now that the 2m dataset
# is so pervasive.
fill_holes(dem)
dem.write_gdal(fn)
except Exception as exc:
print "Failed with exception"
print repr(exc)
def lidar_resamp():
lidar_resamp_fn = 'lidar-levees.tif'
if not os.path.exists(lidar_resamp_fn):
# Will compile the lidar data here:
lidar_levees = field.SimpleGrid(extents=lidar_mask().extents,
F=np.zeros(lidar_mask().F.shape,
np.float32))
lidar_levees.F[:, :] = np.nan
lidar_resamp = lidar_src.to_grid(bounds=lidar_mask().extents,
dx=lidar_mask().dx,
dy=lidar_mask().dy)
lidar_resamp.F[~lidar_mask().F] = np.nan
# There are seams, at least 1px wide, between lidar tiles.
lidar_resamp.fill_by_convolution(iterations=1)
lidar_resamp.write_gdal('lidar-levees.tif')
else:
lidar_resamp = field.GdalGrid('lidar-levees.tif')
return lidar_resamp
"""
The in-pond data is very blocky - apply a bit of smoothing
before it gets used in the dem
"""
import matplotlib.pyplot as plt
from stompy.spatial import field
##
fn = '../../sbsprp/SbayPondBathy2005/merged_ponds.tif'
ponds_dem = field.GdalGrid(fn)
# limit it to the parts we're actually using
ponds_crop = ponds_dem.extract_tile(xxyy=(579160, 592550, 4141950, 4146580),
res=2,
interpolation='bilinear')
# expand a bit
ponds_crop.fill_by_convolution(iterations=5, kernel_size=5)
# Smooth it
ponds_crop.smooth_by_convolution(iterations=5)
##
plt.figure(2).clf()
fig, ax = plt.subplots(num=2)
ponds_crop.plot(ax=ax, interpolation='nearest')
##
# And looks like we need to convert to m, too.
ponds_crop.F *= 0.3048
v04: look at 2018 data, first with an eye to detecting flows.
"""
import pandas as pd
import xarray as xr
from stompy import utils
import numpy as np
import pystan
import matplotlib.pyplot as plt
from stompy.spatial import field
import seawater
from stompy.plot import plot_utils
##
img = field.GdalGrid("../../model/grid/boundaries/junction-bathy-rgb.tif")
##
ds = xr.open_dataset('pings-2018-03-20T20:00_2018-03-20T22:00.nc')
# SM4, SM3
ds = ds.isel(rx=[2, 3])
##
beacons = ds.rx_beacon.values
beacon_to_xy = dict([(ds.rx_beacon.values[i], (ds.rx_x.values[i],
ds.rx_y.values[i]))
for i in range(ds.dims['rx'])])
##
# can get a little bit of constraint from 2-rx pings, but
upper_ll = [-120.930408, 37.309940]
lower_ll = [-121.271098, 37.691794]
release_xy = proj_utils.mapper('WGS84', 'EPSG:26910')([upper_ll, lower_ll])
##
dem_mrg = field.MultiRasterField([
"../../../bathy/junction-composite-20200605-w_smooth.tif",
"/home/rusty/data/bathy_dwr/gtiff/dem_bay_delta_10m_v3_20121109_2.tif"
])
clip = (646390., 648336., 4184677., 4187210.)
dem = dem_mrg.extract_tile(clip)
##
aerial = field.GdalGrid(
"../../../gis/aerial/m_3712114_sw_10_h_20160621_20161004-UTM.tif")
aerial.F = aerial.F[:, :, :3] # drop alpha - seems mis-scaled
##
#grid=unstructured_grid.UnstructuredGrid.read_ugrid("../../../model/grid/snubby_junction/snubby-06.nc")
grid = unstructured_grid.UnstructuredGrid.read_ugrid(
"../../../model/suntans/snubby-08-edit60-with_bathysmooth.nc")
grid_poly = grid.boundary_polygon()
##
rx_locs = pd.read_csv("../../../field/hor_yaps/yap-positions.csv")
##
# 03: fix initial condition
# 04: conveyance2d=-1
# 05: bedlevtype=6
model.set_run_dir("runs/hypso_05", mode='askclobber')
model.run_start = np.datetime64('2017-12-01')
model.run_stop = np.datetime64('2017-12-10')
model.load_mdu('template.mdu')
src_grid = '../grid/CacheSloughComplex_v111-edit19.nc'
dst_grid = os.path.basename(src_grid).replace('.nc', '-bathy.nc')
bathy_fn = "../bathy/merged_2m-20181113.tif"
if utils.is_stale(dst_grid, [src_grid, bathy_fn]):
g = unstructured_grid.UnstructuredGrid.from_ugrid(src_grid)
dem = field.GdalGrid(bathy_fn)
node_depths = dem(g.nodes['x'])
while np.any(np.isnan(node_depths)):
missing = np.nonzero(np.isnan(node_depths))[0]
print("Looping to fill in %d missing node depths" % len(missing))
for n in missing:
nbrs = g.node_to_nodes(n)
node_depths[n] = np.nanmean(node_depths[nbrs])
g.add_node_field('depth', node_depths, on_exists='overwrite')
g.write_ugrid(dst_grid, overwrite=True)
else:
g = unstructured_grid.UnstructuredGrid.from_ugrid(dst_grid)
model.set_grid(g)
# I think this doesn't matter with conveyance2d>0
# 6 means cells=mean(nodes), edges=shallower(cells)
import os
from stompy.grid import unstructured_grid
from stompy.spatial import field
from stompy import utils
import matplotlib.pyplot as plt
import numpy as np
##
bathy_dir = "../../../bathy"
bathy_existing_fn = os.path.join(bathy_dir,
'compiled-dem-existing-20210608-1m.tif')
dem = field.GdalGrid(bathy_existing_fn)
# grid_dir="../../../grid"
# grid_fn=os.path.join(grid_dir,'quad_tri_v21-edit05.nc')
# g=unstructured_grid.UnstructuredGrid.read_ugrid(grid_fn)
# g.renumber()
##
g = unstructured_grid.UnstructuredGrid.read_ugrid(
'pesca_butano_existing_deep_bathy.nc')
##
alpha = np.linspace(0, 1, 5)
edge_data = np.zeros((g.Nedges(), 3), np.float64)
import numpy as np
import matplotlib.pyplot as plt
from matplotlib import gridspec
import track_common
from statsmodels.graphics.tsaplots import plot_pacf, plot_acf
from statsmodels.tsa import stattools
from stompy import nanpsd
from stompy import utils
from stompy.spatial import field
import seaborn as sns
import stompy.plot.cmap as scmap
turbo = scmap.load_gradient('turbo.cpt')
##
dem = field.GdalGrid("../../bathy/junction-composite-dem-no_adcp.tif")
clip = (647167.570005404, 647416.10024743, 4185000, 4186500)
demcrop = dem.crop(clip)
##
def unwrap(angles):
deltas = (np.diff(angles) + np.pi) % (2 * np.pi) - np.pi
return np.cumsum(np.r_[angles[0], deltas])
def expand(data, time, dt_nom=5.0):
# fill, based on nominal tag interval of 5 seconds
di = np.round(np.diff(time) / dt_nom).astype(np.int32)
assert di.min() > 0
from shapely import geometry
from scipy import ndimage
from stompy.spatial import (field, wkb2shp, linestring_utils)
from stompy import utils
import stompy.plot.cmap as scm
from stompy.grid import unstructured_grid
from stompy.model.delft import io
data_root = '/opt/data'
##
dem = field.GdalGrid(
os.path.join(data_root,
'bathy_interp/master2017/tiles_2m_20171024/merged_2m.tif'))
##
# More details on the provenance of this on the Google drive in
# Modeling/Lower South Bay/shoreline elevs
inv_fn = "linear_features-v02.shp"
inv = wkb2shp.shp2geom(inv_fn)
##
# For the moment, focus on just the parts of the inventory which overlap with the DEM.
bounds_xyxy = np.array(dem.extents)[[0, 2, 1, 3]]
box = geometry.box(*bounds_xyxy)
def base_config(model):
"""
model: Model instance, should already have run_dir set.
"""
model.num_procs = 4
model.z_datum = 'NAVD88'
model.projection = 'EPSG:26910'
model.utc_offset = np.timedelta64(-8, 'h') # PST
model.run_start = np.datetime64('2017-12-01')
# very short!
model.run_stop = np.datetime64('2017-12-05')
model.load_mdu('template.mdu')
src_grid = '../grid/CacheSloughComplex_v111-edit19fix.nc'
dst_grid = os.path.basename(src_grid).replace('.nc', '-bathy.nc')
bathy_fn = "../bathy/merged_2m-20181113.tif"
if utils.is_stale(dst_grid, [src_grid, bathy_fn]):
g = unstructured_grid.UnstructuredGrid.from_ugrid(src_grid)
dem = field.GdalGrid(bathy_fn)
# node_depths=dem(g.nodes['x'])
import dem_cell_node_bathy
node_depths = dem_cell_node_bathy.dem_to_cell_node_bathy(dem, g)
g.add_node_field('depth', node_depths, on_exists='overwrite')
g.write_ugrid(dst_grid, overwrite=True)
else:
g = unstructured_grid.UnstructuredGrid.from_ugrid(dst_grid)
model.set_grid(g)
# I think this doesn't matter with conveyance2d>0
model.mdu['geometry', 'BedlevType'] = 6
# ostensibly analytic 2D conveyance with 3.
# 2 is faster, and appears less twitchy while showing very similar
# calibration results.
# 0 blocks some flow, 1 was a little twitchy.
model.mdu['geometry', 'Conveyance2D'] = -1
# enabling this seems to cause a lot of oscillation.
model.mdu['geometry', 'Nonlin2D'] = 0
model.mdu['output', 'MapInterval'] = 7200
#model.mdu['output','WaqInterval']=1800
model.mdu['physics', 'UnifFrictCoef'] = 0.025
model.set_cache_dir('cache')
model.add_gazetteer('gis/model-features.shp')
model.add_gazetteer('gis/point-features.shp')
hm.SourceSinkBC.dredge_depth = -1
hm.FlowBC.dredge_depth = -1
# check_bspp.py has the code that converted original tim to csv.
# awkward reloading of that, but at least it's independent of the
# simulation period.
model.add_bcs(barker_data.BarkerPumpsBC(name='Barker_Pumping_Plant'))
# Decker only exists post-2015
if model.run_start > np.datetime64("2015-11-16"):
model.add_bcs(
hm.NwisStageBC(name='decker', station=11455478, cache_dir='cache'))
else:
# maybe fall back to Rio Vista, or some adjustment thereof
raise Exception(
"Decker tidal data starts 2015-11-16, too late for this simulation period"
)
model.add_bcs(
hm.NwisFlowBC(name='threemile', station=11337080, cache_dir='cache'))
# GSS: from compare_flows and lit, must be flipped.
model.add_bcs(
hm.NwisFlowBC(name='Georgiana',
station=11447903,
cache_dir='cache',
filters=[hm.Transform(lambda x: -x)]))
model.add_bcs(
hm.NwisFlowBC(name='dcc',
station=11336600,
cache_dir='cache',
filters=[FillGaps(),
hm.Transform(lambda x: -x)]))
sac = hm.NwisFlowBC(
name="SacramentoRiver",
station=11447650,
pad=np.timedelta64(5, 'D'),
cache_dir='cache',
filters=[hm.LowpassGodin(),
hm.Lag(np.timedelta64(-2 * 3600, 's'))])
model.add_bcs(sac)
##
pad = np.timedelta64(5, 'D')
lisbon_ds = cdec.cdec_dataset(station='LIS',
start_date=model.run_start - pad,
end_date=model.run_stop + pad,
sensor=20,
cache_dir=cache_dir)
# to m3/s
lisbon_ds['Q'] = lisbon_ds['sensor0020'] * 0.028316847
# to hourly average
lisbon_ds.groupby(lisbon_ds.time.astype('M8[h]')).mean()
lisbon_bc = hm.FlowBC(name='lis', Q=lisbon_ds.Q)
# Roughness - handled by caller.
# model.add_RoughnessBC(shapefile='forcing-data/manning_n.shp')
# Culvert at CCS
if 1:
# rough accounting of restoration
if model.run_start < np.datetime64("2014-11-01"):
# name => id
# polylinefile is filled in from shapefile and name
model.add_Structure(
name='ccs_breach',
type='gate',
door_height=15, # no overtopping?
lower_edge_level=0.89,
opening_width=0.0, # pretty sure this is ignored.
sill_level=0.85, # gives us a 0.05m opening?
horizontal_opening_direction='symmetric')
# these are really just closed
for levee_name in ['ccs_west', 'ccs_east']:
model.add_Structure(name=levee_name,
type='gate',
door_height=15,
lower_edge_level=3.5,
opening_width=0.0,
sill_level=3.5,
horizontal_opening_direction='symmetric')
mon_sections = model.match_gazetteer(monitor=1, geom_type='LineString')
mon_points = model.match_gazetteer(geom_type='Point')
model.add_monitor_sections(mon_sections)
model.add_monitor_points(mon_points)
return model
def dem():
return field.GdalGrid(
os.path.join(
here, "../../../bathy/junction-composite-20200604-w_smooth.tif"))
def set_lsb_bathy(grid):
check_bathy()
# First, load in the original sfb_dfm grid to get bathymetry
log.info("Loading SFB DFM v2 grid")
sfb_dfm_grid=xr.open_dataset('sfb_dfm/sfei_v19_net.nc')
sfb_X=np.c_[ sfb_dfm_grid.NetNode_x.values,
sfb_dfm_grid.NetNode_y.values]
sfb_dfm_field=field.XYZField(X=sfb_X,
F=sfb_dfm_grid.NetNode_z.values)
lsb_X=grid.nodes['x']
# This will set all points within the convex hull of the original
# grid. These elevations are used in the output (a) outside the
# LSB merged_2m DEM, and (b) to prioritize which nodes to use when
# moving depths from edges to nodes.
# Could argue that it would be better to pull point elevations here
# from the DEM where they overlap. Probably makes very little difference
lsb_z=sfb_dfm_field(lsb_X) # still some nans at this point
#
log.info("Loading LSB dem from %s"%merged_2m_path)
dem=field.GdalGrid(merged_2m_path)
#
# Use the 2m DEM to find an effective minimum depth for each edge covered
# by the DEM.
edge_min_depths=depth_connectivity.edge_connection_depth(grid,dem,centers='lowest')
# move those edge depths to node depths
node_depths=depth_connectivity.greedy_edgemin_to_node(grid,lsb_z,edge_min_depths)
# Still have some nodes with nan depth, first try filling in with the DEM.
missing=np.isnan(node_depths)
node_depths[missing]=dem( lsb_X[missing,:] )
# And wrap it up with a more forgiving interpolation from the original sfb_dfm_v2
# grid (about 12 points on the convex hull)
still_missing=np.isnan(node_depths)
node_depths[still_missing]=sfb_dfm_field.interpolate( lsb_X[still_missing,:],
'nearest' )
assert np.isnan(node_depths).sum()==0
# Update the grid
grid.nodes['depth']=node_depths
if 0: # caller is going to deal with I/O
out_file='lsb_v99_bathy_net.nc'
os.path.exists(out_file) and os.unlink(out_file)
dfm_grid.write_dfm(grid,out_file)
if 0: # plot for development.
plt.figure(10).clf()
fig,ax=plt.subplots(num=10)
edge_mins=grid.nodes['depth'][grid.edges['nodes']].min(axis=1)
ecoll=grid.plot_edges(lw=1.5,values=edge_mins)
ncoll=grid.plot_nodes(values=grid.nodes['depth'])
plt.setp([ecoll,ncoll],clim=[-3,3])
plot_utils.cbar(ncoll,extras=[ecoll])
# Modified in place, but return just in case
return grid # QED.
def bathy():
from stompy.spatial import field
return field.GdalGrid(
'../bathy/OldRvr_at_SanJoaquinRvr2012_0104-utm-m_NAVD.tif')
if gen_grids:
grid_dir = "../../../grid"
grid_fn = os.path.join(grid_dir, 'quad_tri_v21-edit08.nc')
g = unstructured_grid.UnstructuredGrid.read_ugrid(grid_fn)
g.renumber()
if gen_weirs:
lines = wkb2shp.shp2geom('line_features.shp')
for dem_fn, name in dems:
if not os.path.exists(dem_fn):
print("---- DEM file %s doesn't exist ----" % dem_fn)
continue
print("Loading DEM for %s" % name)
dem = field.GdalGrid(dem_fn)
if gen_weirs:
print("Generating fixed weirs")
fixed_weir_fn = "fixed_weirs-%s.pliz" % name
fixed_weirs = [] # suitable for write_pli
dx = 5.0 # m. discretize lines at this resolution
for i in range(len(lines)):
feat = lines[i]
if feat['type'] != 'fixed_weir': continue
print(f"Processing levee feature {feat['name']}")
xy = np.array(feat['geom'])
xy = linestring_utils.upsample_linearring(xy,
dx,
closed_ring=False)
def coamps_dataset(bounds,
start,
stop,
cache_dir,
fields=['wnd_utru', 'wnd_vtru', 'pres_msl'],
missing='omit',
fetch=True):
"""
Downloads COAMPS winds for the given period (see fetch_coamps_wind),
trims to the bounds xxyy, and returns an xarray Dataset.
fields: these are the coamps names for the fields. for legacy reasons,
some of them are remapped in the dataset
wnd_utru => wind_u
wnd_vtru => wind_v
pres_msl => pres
missing:'omit' - if any of the fields are missing for a timestep, omit that timestep.
None: will raise an error when GdalGrid fails.
may in the future also have 'nan' which would fill that variable with nan.
fetch: if False, rely on cached data.
"""
fields = list(fields)
fields.sort()
if fetch:
fetch_coamps_wind(start, stop, cache_dir, fields=fields)
pad = 10e3
crop = [bounds[0] - pad, bounds[1] + pad, bounds[2] - pad, bounds[3] + pad]
dss = []
renames = {'wnd_utru': 'wind_u', 'wnd_vtru': 'wind_v', 'pres_msl': 'pres'}
for recs in coamps_files(start, stop, cache_dir, fields=fields):
timestamp = recs[fields[0]]['timestamp']
timestamp_dt = utils.to_datetime(timestamp)
timestamp_str = timestamp_dt.strftime('%Y-%m-%d %H:%M')
# use the local file dirname to get the same model subdirectory
# i.e. cencoos_4km
cache_fn = os.path.join(
os.path.dirname(recs[fields[0]]['local']), "%s-%s.nc" %
(timestamp_dt.strftime('%Y%m%d%H%M'), "-".join(fields)))
if not os.path.exists(cache_fn):
print(timestamp_str)
ds = xr.Dataset()
ds['time'] = timestamp
# check to see that all files exists:
# this will log the missing urls, which can then be added to the missing
# list above if they are deemed really missing
missing_urls = [
recs[field_name]['url'] for field_name in fields
if not os.path.exists(recs[field_name]['local'])
]
if len(missing_urls) > 0 and missing == 'omit':
log.warning("Missing files for time %s: %s" %
(timestamp_str, ", ".join(missing_urls)))
continue
for i, field_name in enumerate(fields):
# load the grib file
raw = field.GdalGrid(recs[field_name]['local'])
# Reproject to UTM: these come out as 3648m resolution, compared to 4km input.
# Fine. 366 x 325. Crops down to 78x95
raw_utm = raw.warp("EPSG:26910").crop(crop)
if i == 0: # take spatial details from the first field
x, y = raw_utm.xy()
ds['x'] = ('x', ), x
ds['y'] = ('y', ), y
ds_name = renames.get(field_name, field_name)
# copy, in hopes that we can free up ram more quickly
ds[ds_name] = ('y', 'x'), raw_utm.F.copy()
ds.to_netcdf(cache_fn)
ds.close()
ds = xr.open_dataset(cache_fn)
ds.load() # force load of data
ds.close() # and close out file handles
dss.append(ds) # so this is all in ram.
ds = xr.concat(dss, dim='time')
return ds
date_str = "20201230"
clip = zoom = (551800, 553290, 4124100, 4125400)
# cbec surfaces:
# For starters load just the N marsh and pond portion.
# For compilation at the end load the whole thing
if version == 'asbuilt':
cbec_dem_fn = '../data/cbec/to_RCD/asbuilt/merged.tif'
cbec_dem_name = "cbec As Built"
elif version == 'existing':
cbec_dem_fn = '../data/cbec/to_RCD/existing_grade/merged.tif'
cbec_dem_name = "cbec Existing Grade"
cbec_dem = field.GdalGrid(cbec_dem_fn, geo_bounds=clip)
cbec_full_dem = field.GdalGrid(cbec_dem_fn)
cbec_full_dem.name = cbec_dem.name = cbec_dem_name
# as_built_extents,res = field.GdalGrid.metadata('../data/cbec/to_RCD/asbuilt/merged.tif')
# existing_dem = field.GdalGrid('../data/cbec/to_RCD/existing_grade/merged.tif',geo_bounds=clip)
# existing_dem.name="cbec Existing Grade"
# 2017 CoNED LIDAR
lidar2017 = field.GdalGrid(
'../data/noaa/lidar/2017CoNED-north_marsh-lidar/Job558295_cent_ca_coned.tif',
geo_bounds=clip)
lidar2017.name = "CoNED 2017 LiDaR"
lidar2011 = field.GdalGrid(
'../data/noaa/lidar/2011CoastCons-north_marsh-lidar/Job558340_CA2009_coastal_DEM.tif',
geo_bounds=clip)
from scipy import ndimage as ndi
from stompy.spatial import field
import numpy as np
import matplotlib.pyplot as plt
import os
opj = os.path.join
IDRIVE = '/media/idrive'
##
fn = '../../sbsprp/SbayPondBathy2005/merged_ponds.tif'
ponds_dem = field.GdalGrid(fn)
zoom = ponds_dem.extents
if 1:
fn = opj(
IDRIVE,
'BASELAYERS/Elevation_DerivedProducts/LiDAR 2005-2012 entire Bay Area from AECOM',
'USGS_TopoBathy/San_Francisco_TopoBathy_Elevation_2m.tif')
geo_bounds = ponds_dem.extents # [550000,560000,4.14e6,4.14e6+10e3]
dem = field.GdalGrid(fn, geo_bounds=geo_bounds)
else:
dem = field.GdalGrid('tiles_2m_20170508/merged_2m.tif', geo_bounds=zoom)
##
plt.figure(1).clf()
fig, ax = plt.subplots(num=1)
##
bounds = wkb2shp.shp2geom('region-bounds-v00.shp')
##
from shapely import ops
one_poly = ops.cascaded_union(bounds['geom'])
bounds_xyxy = one_poly.bounds
##
dem = field.GdalGrid(
('/media/idrive/BASELAYERS/Elevation_DerivedProducts/'
'LiDAR 2005-2012 entire Bay Area from AECOM/USGS_TopoBathy/'
'San_Francisco_TopoBathy_Elevation_2m.tif'),
geo_bounds=[
bounds_xyxy[0], bounds_xyxy[2], bounds_xyxy[1], bounds_xyxy[3]
])
##
plt.figure(1).clf()
fig, ax = plt.subplots(num=1)
# DEM is (3189, 3544)
# Restricting to elevation below 2.155, still 8M pixels.
img = dem.downsample(4).plot(vmin=-8,
vmax=5.0,
cmap='gray',
interpolation='nearest',
ax=ax)
coords=dict(time=[
model.run_start - 24 * h, model.run_start, model.run_start +
ramp_hours * h, model.run_stop, model.run_stop + 24 * h
]))
Q_downstream = drv.FlowBC(name='SJ_downstream', Q=Qdown)
h_old_river = drv.StageBC(name='Old_River', z=2.2 - 0.65)
model.add_bcs([Q_upstream, Q_downstream, h_old_river])
model.write()
#--
if 0:
# Add in variable roughness
ec = model.grid.edges_center()
z0_map = field.GdalGrid("../../bathy/composite-roughness.tif")
z0 = z0_map(ec)
z0[np.isnan(z0)] = float(model.config['z0B'])
model.ic_ds['z0B'] = ('time', 'Ne'), z0[None, :]
model.write_ic_ds()
#
model.partition()
model.sun_verbose_flag = '-v'
model.run_simulation()
# with the lower friction, get a vertical courant number of 5.25
# due to a cell that's 1mm thick.
