def set_time(t_idx):
Gvar[:] = merge(
lambda ds: ds['sa1'].isel(time=t_idx).isel(laydim=0).values)
depth = merge(lambda ds: ds['waterdepth'].isel(time=t_idx).values)
coll.set_array(np.ma.array(Gvar[cell_mask], mask=depth[cell_mask] < 0.05))
t = dss[0].time.values[t_idx]
date_txt.set_text(utils.to_datetime(t).strftime('%Y-%m-%d %H:%M'))
def __init__(self,grid,data,start,end,output,t_ref,**kw):
self.grid=grid
self.data=data
self.start=start
self.end=end
self.output=output
self.t_ref=t_ref
# optional arguments
utils.set_keywords(self,kw)
# Do the deed
# DWAQ segment files are written as binary, with each time step as:
# <4-byte little-endian time in seconds since reference time>
# <4-byte little-endian floating> * N_segments
# data is in segment order, all surface segments, then all segments on the
# next layer down, and so on.
with open(output,'wb') as fp:
for t in self.time_steps():
log.info("Processing %s"%( utils.to_datetime(t).strftime('%Y-%m-%d %H:%M')))
t_sec=(t-self.t_ref)/np.timedelta64(1,'s')
t_sec=np.array([t_sec],'<i4')
data2d=self.field_2d(t)
assert np.all(np.isfinite(data2d)),"Error -- getting some non-finite values"
data3d=self.extrude_to_3d(data2d)
fp.write( t_sec.tobytes() )
fp.write(data3d.astype('<f4'))
self.post_frame(t,data2d,data3d)
def post_frame(self,t,data2d,data3d):
if self.plot_mode is None:
return
import matplotlib.pyplot as plt
import stompy.plot.cmap as scmap
if self.fig is None:
self.fig=plt.figure()
ax=self.fig.add_subplot(1,1,1)
# min/max over entire period
#vmin=self.data['value'].min()
#vmax=self.data['value'].max()
# min/max for this moment
vmin=data2d.min()
vmax=data2d.max()
cmap=scmap.load_gradient('turbo.cpt')
self.coll=self.grid.plot_cells(values=data2d,clim=[vmin,vmax],
lw=0.5,
cmap=cmap,ax=ax)
self.coll.set_edgecolor('face')
ax.axis('equal')
plt.colorbar(self.coll,ax=ax)
self.txt=ax.text(0.05,0.05,"text",transform=ax.transAxes)
else:
self.coll.set_array(data2d)
self.txt.set_text( str(t) )
if self.plot_mode=='each':
self.fig.canvas.draw()
plt.pause(0.01)
elif self.plot_mode=='save':
time_str=utils.to_datetime(t).strftime('%Y%m%dT%H%M')
self.fig.savefig('extrap-%s.png'%time_str)
def noaa_cached(noaa_station, product, start_date, end_date, *a, **k):
start_dt = utils.to_datetime(start_date)
end_dt = utils.to_datetime(end_date)
cache_fn = os.path.join(
cache_dir,
"%s-%s-%s-%s.nc" % (noaa_station, product, start_dt.strftime("%Y%m%d"),
end_dt.strftime("%Y%m%d")))
if not os.path.exists(cache_fn):
dat = noaa_coops.coops_dataset_product(station=noaa_station,
product=product,
start_date=start_date,
end_date=end_date,
days_per_request=31)
dat.to_netcdf(cache_fn)
# even if newly fetched, read from disk to avoid non-reproducible behavior.
return xr.open_dataset(cache_fn)
def __init__(self, fig, t, **kw):
self.__dict__.update(kw)
snap_data = extract_particle_snapshot(t)
if self.ax is None:
fig.clf()
ax = fig.add_subplot(1, 1, 1)
ax.set_position([0, 0, 1, 1])
else:
ax = self.ax
plot_wkb.plot_polygon(grid_poly,
ax=ax,
fc='0.8',
ec='none',
lw=0.5,
zorder=-2)
plot_wkb.plot_polygon(grid_poly,
ax=ax,
fc='none',
ec='k',
lw=0.5,
zorder=2)
station_xys = np.array([station.xy.values for station in stations])
ax.plot(station_xys[:, 0], station_xys[:, 1], 'ko', ms=5)
values = self.snap_to_value(snap_data)
sel = np.isfinite(values)
stale_s = np.abs(snap_data['fill_dist'])
sel = sel & (stale_s < self.stale_thresh_s)
scat = ax.scatter(snap_data['x'][sel], snap_data['y'][sel],
40 * (1 / (1 + stale_s[sel] / 1800.)), values[sel])
scat.set_cmap(self.cmap)
scat.set_clim(self.clim)
pos = ax.get_position()
cax = fig.add_axes([
pos.xmin + 0.08, pos.ymin + 0.93 * pos.height, pos.width * 0.25,
0.02
])
ax.text(0.08,
0.96,
utils.to_datetime(t).strftime("%Y-%m-%d %H:%M PST"),
transform=ax.transAxes)
plt.colorbar(scat,
cax=cax,
orientation='horizontal',
label=self.units_label)
ax.xaxis.set_visible(0)
ax.yaxis.set_visible(0)
ax.axis('equal')
ax.axis(self.zoom)
def coamps_press_windxy_dataset(g_target, start, stop):
"""
Downloads COAMPS winds for the given period (see fetch_coamps_wind),
trims to the bounds of g_target, and returns an xarray Dataset.
"""
fetch_coamps_wind(start, stop)
xy_min = g_target.nodes['x'].min(axis=0)
xy_max = g_target.nodes['x'].max(axis=0)
pad = 10e3
crop = [xy_min[0] - pad, xy_max[0] + pad, xy_min[1] - pad, xy_max[1] + pad]
dss = []
for recs in coamps_files(start, stop):
timestamp = recs['wnd_utru']['timestamp']
timestamp_dt = utils.to_datetime(timestamp)
timestamp_str = timestamp_dt.strftime('%Y-%m-%d %H:%M')
# use the local file dirname to get the same model subdirectory
# i.e. cencoos_4km
cache_fn = os.path.join(os.path.dirname(recs['pres_msl']['local']),
"%s.nc" % timestamp_dt.strftime('%Y%m%d%H%M'))
if not os.path.exists(cache_fn):
print(timestamp_str)
# load the 3 fields:
wnd_utru = field.GdalGrid(recs['wnd_utru']['local'])
wnd_vtru = field.GdalGrid(recs['wnd_vtru']['local'])
pres_msl = field.GdalGrid(recs['pres_msl']['local'])
# Reproject to UTM: these come out as 3648m resolution, compared to 4km input.
# Fine. 366 x 325. Crops down to 78x95
wnd_utru_utm = wnd_utru.warp("EPSG:26910").crop(crop)
wnd_vtru_utm = wnd_vtru.warp("EPSG:26910").crop(crop)
pres_msl_utm = pres_msl.warp("EPSG:26910").crop(crop)
ds = xr.Dataset()
ds['time'] = timestamp
x, y = wnd_utru_utm.xy()
ds['x'] = ('x', ), x
ds['y'] = ('y', ), y
# copy, in hopes that we can free up ram more quickly
ds['wind_u'] = ('y', 'x'), wnd_utru_utm.F.copy()
ds['wind_v'] = ('y', 'x'), wnd_vtru_utm.F.copy()
ds['pres'] = ('y', 'x'), pres_msl_utm.F.copy()
ds.to_netcdf(cache_fn)
ds.close()
ds = xr.open_dataset(cache_fn)
ds.load() # force load of data
ds.close() # and close out file handles
dss.append(ds) # so this is all in ram.
ds = xr.concat(dss, dim='time')
return ds
def set_time(t_idx):
s_surf = merge(
lambda ds: ds.sa1.isel(time=t_idx).isel(laydim=0).values)
s_bed = merge(
lambda ds: ds.sa1.isel(time=t_idx).isel(laydim=-1).values)
depth = merge(lambda ds: ds['waterdepth'].isel(time=t_idx).values)
Gvar[:] = (s_surf - s_bed) / depth.clip(1, np.inf)
coll.set_array(
np.ma.array(Gvar[cell_mask], mask=depth[cell_mask] < 0.05))
t = dss[0].time.values[t_idx]
date_txt.set_text(utils.to_datetime(t).strftime('%Y-%m-%d %H:%M'))
def __init__(self, **kw):
utils.set_keywords(self, kw)
if self.base_path is None:
self.base_path = self.calc_base_path()
if self.start_offset != np.timedelta64(75, 'D'):
# yuck. dates.
yyyymmdd = utils.to_datetime(
utils.to_dt64(self.hydro.t_dn[0]) +
self.start_offset).strftime('%Y%m%d')
self.base_path += "_%s" % (yyyymmdd)
log.info("base_path defaults to %s" % self.base_path)
def usgs_salinity_time_series(station):
# A little tricky - there are two elevations, which have the same parameter
# code of 95 for specific conductance, but ts_id's of 14739 and 14741.
# requesting the parameter once does return an RDB with both in there.
time_labels = [
utils.to_datetime(t).strftime('%Y%m%d') for t in [t_start, t_stop]
]
cache_fn = "usgs%s-%s_%s-salinity.nc" % (station, time_labels[0],
time_labels[1])
if not os.path.exists(cache_fn):
# 95: specific conductance
# 90860: salinity
ds = usgs_nwis.nwis_dataset(station,
t_start,
t_stop,
products=[95, 90860],
days_per_request=20)
usgs_nwis.add_salinity(ds)
ds.to_netcdf(cache_fn)
ds.close()
##
ds = xr.open_dataset(cache_fn)
ds.attrs['lon'] = station_locs[station]['lon']
ds.attrs['lat'] = station_locs[station]['lat']
ds.salinity.attrs['elev_mab'] = station_locs[station]['elev_mab'][0]
if 'salinity_01' in ds:
ds.salinity_01.attrs['elev_mab'] = station_locs[station]['elev_mab'][1]
xy = ll2utm([ds.attrs['lon'], ds.attrs['lat']])
ds.attrs['x'] = xy[0]
ds.attrs['y'] = xy[1]
return ds
def process_period(start_time_string,
end_time_string,
outfileprefix,
force=False):
nc_fn = outfileprefix + ".nc"
print("Processing %s to %s output to %s" %
(start_time_string, end_time_string, nc_fn))
if (not force) and os.path.exists(nc_fn):
print("File %s exists - skipping" % nc_fn)
return
# pick interpolation method (natural neighbor or linear is recommended):
# 'nearest' = nearest neighbor
# 'linear' = linear
# 'cubic' = cubic spline
# 'natural' = natural neighbor
interp_method = 'natural'
# specify comment string (one line only, i.e., no '\n') for *.amu/*.amv files
commentstring = 'Prepared by Allie King, SFEI, times are in PST (ignore the +00:00), adapted by Rusty Holleman'
# specify properties of the wind grid -- this one was used for CASCaDE and sfb_dfm
bounds = [340000, 610000, 3980000, 4294000]
dx = 1500.
dy = 1500.
#--------------------------------------------------------------------------------------#
# Main Program
#--------------------------------------------------------------------------------------#
n_cols = int(round(1 + (bounds[1] - bounds[0]) / dx))
n_rows = int(round(1 + (bounds[3] - bounds[2]) / dy))
x_llcorner = bounds[0]
y_llcorner = bounds[2]
start_date = np.datetime64(start_time_string)
end_date = np.datetime64(end_time_string)
# specify directory containing the compiled wind observation data and station
# coordinates (SFB_hourly_U10_2011.csv, SFB_hourly_V10_2011.csv, etc...)
windobspath = os.path.join(basedir, 'Compiled_Hourly_10m_Winds/data')
# convert start and end time to datetime object
start_dt = utils.to_datetime(start_date)
end_dt = utils.to_datetime(end_date)
# create a meshgrid corresponding to the CASCaDE wind grid
x_urcorner = x_llcorner + dx * (n_cols - 1)
y_urcorner = y_llcorner + dy * (n_rows - 1)
x = np.linspace(x_llcorner, x_urcorner, n_cols)
# RH: orient y the usual way, not the arcinfo/dfm wind way (i.e. remove flipud)
y = np.linspace(y_llcorner, y_urcorner, n_rows)
xg, yg = np.meshgrid(x, y)
# read the observed wind data
tz_offset = dt.timedelta(hours=8)
try:
# start_time,end_time are in UTC, so remove the offset when requesting data
# from wlib which expects PST
time_days, station_names, U10_obs = wlib.read_10m_wind_data_from_csv(
os.path.join(windobspath, 'SFB_hourly_U10_'), start_dt - tz_offset,
end_dt - tz_offset)
time_days, station_names, V10_obs = wlib.read_10m_wind_data_from_csv(
os.path.join(windobspath, 'SFB_hourly_V10_'), start_dt - tz_offset,
end_dt - tz_offset)
except FileNotFoundError:
print("Okay - probably beyond the SFEI data")
U10_obs = V10_obs = None
if U10_obs is not None:
# note that time_days is just decimal days after start, so it doesn't need to be adjusted for timezone.
# read the coordinates of the wind observation stations
df = pd.read_csv(os.path.join(windobspath, 'station_coordinates.txt'))
station_names_check = df['Station Organization-Name'].values
x_obs = df['x (m - UTM Zone 10N)'].values
y_obs = df['y (m - UTM Zone 10N)'].values
Nstations = len(df)
for snum in range(Nstations):
if not station_names[snum] == station_names_check[snum]:
raise (
'station names in station_coordinates.txt must match headers in SFB_hourly_U10_YEAR.csv and SFB_hourly_V10_YEAR.csv files'
)
else:
x_obs = np.zeros(0, np.float64)
y_obs = np.zeros(0, np.float64)
Nstations = 0
# Fabricate time_days
all_times = []
t = start_dt
interval = dt.timedelta(hours=1)
while t <= end_dt:
all_times.append(t)
t = t + interval
all_dt64 = np.array([utils.to_dt64(t) for t in all_times])
time_days = (all_dt64 - all_dt64[0]) / np.timedelta64(1, 's') / 86400.
# zip the x, y coordinates for use in the griddata interpolation
points = np.column_stack((x_obs, y_obs))
# loop through all times, at each time step find all the non-nan data, and
# interpolate it onto the model grid, then compile the data from all times
# into a dimension-3 matrix. keep track of which stations were non-nan ('good')
# at each time step in the matrix igood
coamps_ds = None # handled on demand below
coamps_xy = None # ditto
# drops COAMPS data points within buffer dist of a good observation
buffer_dist = 30e3
for it in range(len(time_days)):
if it % 10 == 0:
print("%d/%d steps" % (it, len(time_days)))
#-- augment with COAMPS output
target_time = start_date + np.timedelta64(int(time_days[it] * 86400),
's')
if (coamps_ds is None) or (target_time > coamps_ds.time.values[-1]):
coamps_ds = coamps.coamps_dataset(bounds,
target_time,
target_time +
np.timedelta64(1, 'D'),
cache_dir=cache_dir,
fields=['wnd_utru', 'wnd_vtru'])
# reduce dataset size -- out in the ocean really don't need too many points
coamps_ds = coamps_ds.isel(x=slice(None, None, 2),
y=slice(None, None, 2))
coamps_X, coamps_Y = np.meshgrid(coamps_ds.x.values,
coamps_ds.y.values)
coamps_xy = np.c_[coamps_X.ravel(), coamps_Y.ravel()]
print("COAMPS shape: ", coamps_X.shape)
# seems that the coamps dataset is not entirely consistent in its shape?
# not sure what's going on, but best to redefine this each time to be
# sure.
@memoize.memoize()
def mask_near_point(xy):
dists = utils.dist(xy, coamps_xy)
return (dists > buffer_dist)
coamps_time_idx = utils.nearest(coamps_ds.time, target_time)
coamps_sub = coamps_ds.isel(time=coamps_time_idx)
# Which coamps points are far enough from good observations. there are
# also some time where coamps data is missing
# mask=np.ones(len(coamps_xy),np.bool8)
mask = np.isfinite(coamps_sub.wind_u.values.ravel())
# find all non-nan data at this time step
if U10_obs is not None:
igood = np.logical_and(~np.isnan(U10_obs[it, :]),
~np.isnan(V10_obs[it, :]))
obs_xy = np.c_[x_obs[igood], y_obs[igood]]
for xy in obs_xy:
mask = mask & mask_near_point(xy)
input_xy = np.concatenate([obs_xy, coamps_xy[mask]])
input_U = np.concatenate(
[U10_obs[it, igood],
coamps_sub.wind_u.values.ravel()[mask]])
input_V = np.concatenate(
[V10_obs[it, igood],
coamps_sub.wind_v.values.ravel()[mask]])
else:
# No SFEI data --
input_xy = coamps_xy[mask]
input_U = coamps_sub.wind_u.values.ravel()[mask]
input_V = coamps_sub.wind_v.values.ravel()[mask]
if np.any(np.isnan(input_U)) or np.any(np.isnan(input_V)):
import pdb
pdb.set_trace()
Ngood = len(input_xy)
# set the interpolation method to be used in this time step: interp_method_1.
# ideally, this would just be the user-defined interpolation method:
# interp_method. however, if we do not have enough non-nan data to use the
# user-defined method this time step, temporarily revert to the nearest
# neighbor method
if interp_method == 'natural' or interp_method == 'linear' or interp_method == 'cubic':
if Ngood >= 4:
interp_method_1 = interp_method
else:
interp_method_1 = 'nearest'
elif interp_method == 'nearest':
interp_method_1 = 'nearest'
# if natural neighbor method, interpolate using the pyngl package
if interp_method_1 == 'natural':
U10g = np.transpose(
ngl.natgrid(input_xy[:, 0], input_xy[:, 1], input_U, xg[0, :],
yg[:, 0]))
V10g = np.transpose(
ngl.natgrid(input_xy[:, 0], input_xy[:, 1], input_V, xg[0, :],
yg[:, 0]))
# for other interpolation methods use the scipy package
else:
U10g = griddata(input_xy,
input_U, (xg, yg),
method=interp_method_1)
V10g = griddata(input_xy,
input_V, (xg, yg),
method=interp_method_1)
# since griddata interpolation fills all data outside range with nan, use
# the nearest neighbor method to extrapolate
U10g_nn = griddata(input_xy, input_U, (xg, yg), method='nearest')
V10g_nn = griddata(input_xy, input_V, (xg, yg), method='nearest')
ind = np.isnan(U10g)
U10g[ind] = U10g_nn[ind]
ind = np.isnan(V10g)
V10g[ind] = V10g_nn[ind]
# compile results together over time
# igood_all not updated for COAMPS, omit here.
if it == 0:
U10g_all = np.expand_dims(U10g, axis=0)
V10g_all = np.expand_dims(V10g, axis=0)
# igood_all = np.expand_dims(igood,axis=0)
else:
U10g_all = np.append(U10g_all,
np.expand_dims(U10g, axis=0),
axis=0)
V10g_all = np.append(V10g_all,
np.expand_dims(V10g, axis=0),
axis=0)
# igood_all = np.append(igood_all, np.expand_dims(igood,axis=0), axis=0)
##
# Write netcdf:
ds = xr.Dataset()
ds['time'] = ('time', ), start_date + (time_days * 86400).astype(
np.int32) * np.timedelta64(1, 's')
ds['x'] = ('x', ), x
ds['y'] = ('y', ), y
ds['wind_u'] = ('time', 'y', 'x'), U10g_all
ds['wind_v'] = ('time', 'y', 'x'), V10g_all
os.path.exists(nc_fn) and os.unlink(nc_fn)
ds.to_netcdf(nc_fn)
ic_map.tem1, temp_fill_3d)
tem1 = ic_map.tem1
new_tem1 = tem1.where(tem1 != missing_val,
other=temp_fill_3dx)
ic_map['tem1'] = new_tem1
ic_map.to_netcdf(ic_fn, format='NETCDF3_64BIT')
else:
ic_map = xr.open_dataset(ic_fns[0]) # for timestamping
mdu['restart', 'RestartFile'] = 'initial_conditions_map.nc'
# Had some issues when this timestamp exactly lined up with the reference date.
# adding 1 minute works around that, with a minor warning that these don't match
# exactly
restart_time = utils.to_datetime(ic_map.time.values[0] +
np.timedelta64(60, 's')).strftime(
'%Y%m%d%H%M')
mdu['restart', 'RestartDateTime'] = restart_time
else:
# don't set 3D IC:
mdu['restart', 'RestartFile'] = ""
mdu['restart', 'RestartDateTime'] = ""
##
mdu_fn = os.path.join(run_base_dir, run_name + ".mdu")
mdu.write(mdu_fn)
##
partition_mdu(mdu_fn)
'fs',
# 'ddsd',
'src000', # EBDA
'src001', # EBMUD
'src002' # SFPUC
]
for rel_time in rel_times:
end_time = rel_time + np.timedelta64(60, 'D')
if rel_time < models_start_time:
raise Exception("Set of model data starts after release time")
if end_time > models_end_time:
print("Set of model data ends before end_time")
continue # don't exit, in case the rel_times are not chronological
rel_str = utils.to_datetime(rel_time).strftime('%Y%m%d')
for rising_speed in rising_speeds:
run_dir = f"/opt2/sfb_ocean/ptm/all_source/{rel_str}/w{rising_speed}"
if os.path.exists(run_dir):
# This will probably need to get a better test, for when a run
# failed.
print(f"Directory exists {run_dir}, will skip")
continue
cfg = ptm_setup.Config(rel_time=rel_time,
end_time=end_time,
run_dir=run_dir,
model=model,
rising_speeds_mps=[rising_speed],
sources=sources)
def add_delta_inflow(run_base_dir,
run_start,
run_stop,
ref_date,
static_dir,
grid,
dredge_depth,
old_bc_fn,
all_flows_unit=False,
time_offset=None):
"""
Fetch river USGS river flows, add to FlowFM_bnd.ext:
Per Silvia's Thesis:
Jersey: Discharge boundary affected by tides, discharge and temperature taken
from USGS 11337190 SAN JOAQUIN R A JERSEY POINT, 0 salinity
(Note that Dutch Slough should probably be added in here)
Rio Vista: 11455420 SACRAMENTO A RIO VISTA, temperature from DWR station RIV.
0 salinity.
run_base_dir: location of the DFM inputs
run_start,run_stop: target period for therun
static_dir: path to static assets, specifically Jersey.pli and RioVista.pli
grid: UnstructuredGrid instance, to be modified at inflow locations
old_bc_fn: path to old-style boundary forcing file
all_flows_unit: if True, override all flows to be 1 m3 s-1 for model diagnostics
time_offset: pull data from a shifted timeframe. np.timedelta64(-365,'D') will
pull data from a year earlier than the run.
"""
if time_offset is not None:
run_start = run_start + time_offset
run_stop = run_stop + time_offset
ref_date = ref_date + time_offset
pad = np.timedelta64(3, 'D')
start_dt = utils.to_datetime(run_start)
stop_dt = utils.to_datetime(run_stop)
date_range = "%s_%s" % (start_dt.strftime('%Y%m%d'),
stop_dt.strftime('%Y%m%d'))
if 0: # cache here.
# Cache the original data from USGS, then clean it and write to DFM format
jersey_raw_fn = os.path.join(run_base_dir,
'jersey-raw-%s.nc' % date_range)
if not os.path.exists(jersey_raw_fn):
jersey_raw = usgs_nwis.nwis_dataset(
station="11337190",
start_date=run_start - pad,
end_date=run_stop + pad,
products=[
60, # "Discharge, cubic feet per second"
10
], # "Temperature, water, degrees Celsius"
days_per_request=30)
jersey_raw.to_netcdf(jersey_raw_fn, engine='scipy')
else:
jersey_raw = xr.open_dataset(jersey_raw_fn)
rio_vista_raw_fn = os.path.join(run_base_dir,
'rio_vista-raw-%s.nc' % date_range)
if not os.path.exists(rio_vista_raw_fn):
rio_vista_raw = usgs_nwis.nwis_dataset(
station="11455420",
start_date=run_start - pad,
end_date=run_stop + pad,
products=[
60, # "Discharge, cubic feet per second"
10
], # "Temperature, water, degrees Celsius"
days_per_request=30)
rio_vista_raw.to_netcdf(rio_vista_raw_fn, engine='scipy')
else:
rio_vista_raw = xr.open_dataset(rio_vista_raw_fn)
else: # cache in nwis code
jersey_raw = usgs_nwis.nwis_dataset(
station="11337190",
start_date=run_start - pad,
end_date=run_stop + pad,
products=[
60, # "Discharge, cubic feet per second"
10
], # "Temperature, water, degrees Celsius"
days_per_request='M',
cache_dir=common.cache_dir)
rio_vista_raw = usgs_nwis.nwis_dataset(
station="11455420",
start_date=run_start - pad,
end_date=run_stop + pad,
products=[
60, # "Discharge, cubic feet per second"
10
], # "Temperature, water, degrees Celsius"
days_per_request='M',
cache_dir=common.cache_dir)
if 1: # Clean and write it all out
for src_name, source in [('Jersey', jersey_raw),
('RioVista', rio_vista_raw)]:
src_feat = dio.read_pli(
os.path.join(static_dir, '%s.pli' % src_name))[0]
dredge_grid.dredge_boundary(grid, src_feat[1], dredge_depth)
# Add stanzas to FlowFMold_bnd.ext:
for quant, suffix in [('dischargebnd', '_flow'),
('salinitybnd', '_salt'),
('temperaturebnd', '_temp')]:
with open(old_bc_fn, 'at') as fp:
lines = [
"QUANTITY=%s" % quant,
"FILENAME=%s%s.pli" % (src_name, suffix), "FILETYPE=9",
"METHOD=3", "OPERAND=O", ""
]
fp.write("\n".join(lines))
feat_suffix = dio.add_suffix_to_feature(src_feat, suffix)
dio.write_pli(
os.path.join(run_base_dir,
'%s%s.pli' % (src_name, suffix)),
[feat_suffix])
# Write the data:
if quant == 'dischargebnd':
da = source.stream_flow_mean_daily
da2 = utils.fill_tidal_data(da)
if all_flows_unit:
da2.values[:] = 1.0
else:
# convert ft3/s to m3/s
da2.values[:] *= 0.028316847
elif quant == 'salinitybnd':
da2 = source.stream_flow_mean_daily.copy(deep=True)
da2.values[:] = 0.0
elif quant == 'temperaturebnd':
da = source.temperature_water
da2 = utils.fill_tidal_data(
da) # maybe safer to just interpolate?
if all_flows_unit:
da2.values[:] = 20.0
df = da2.to_dataframe().reset_index()
df['elapsed_minutes'] = (df.time.values -
ref_date) / np.timedelta64(60, 's')
columns = ['elapsed_minutes', da2.name]
if len(feat_suffix) == 3:
node_names = feat_suffix[2]
else:
node_names = [""] * len(feat_suffix[1])
for node_idx, node_name in enumerate(node_names):
# if no node names are known, create the default name of <feature name>_0001
if not node_name:
node_name = "%s%s_%04d" % (src_name, suffix,
1 + node_idx)
tim_fn = os.path.join(run_base_dir, node_name + ".tim")
df.to_csv(tim_fn,
sep=' ',
index=False,
header=False,
columns=columns)
def fmt(d):
return utils.to_datetime(d).strftime("%Y%m%dT%H%M")
##
def roms_davg(val):
dim = val.get_axis_num('depth')
dz = utils.center_to_interval(val.depth.values)
weighted = np.nansum((val * dz).values, axis=dim)
unit = np.sum(np.isfinite(val) * dz, axis=dim)
return weighted / unit
# Compare tides at point reyes:
_, t_start, t_stop = mdu.time_range()
pr_tides_noaa_fn = "noaa_9415020-%s_%s.nc" % (utils.to_datetime(
t_start).strftime('%Y%m%d'), utils.to_datetime(t_stop).strftime('%Y%m%d'))
if not os.path.exists(pr_tides_noaa_fn):
ds = noaa_coops.coops_dataset("9415020", t_start, t_stop, ["water_level"])
ds.to_netcdf(pr_tides_noaa_fn)
ds.close()
pr_tides_noaa = xr.open_dataset(pr_tides_noaa_fn)
feat = obs_pnts[np.nonzero(obs_pnts['name'] == 'NOAA_PointReyes')[0][0]]
xy = np.array(feat['geom'])
ll = utm2ll(xy)
ll_idx = [
np.searchsorted(src.lon % 360, ll[0] % 360),
np.searchsorted(src.lat, ll[1])
]
##
# For restarts, we don't yet know the run_start
while True:
if last_run_dir is not None:
last_run=drv.SuntansModel.load(last_run_dir)
# Have to be careful that run is long enough to output new
# restart time step, otherwise we'll get stuck.
run_start=last_run.restartable_time()
if run_start>=multi_run_stop:
break
if run_interval is not None:
run_stop=min(run_start+run_interval,multi_run_stop)
else:
run_stop=multi_run_stop
print("Simulation period: %s -- %s"%(run_start,run_stop))
date_str=utils.to_datetime(run_start).strftime('%Y%m%d')
# cfg000: first go
# cfg001: 2D
# cfg002: 3D
# cfg003: fix grid topology in suntans, and timesteps
# cfg004: run in one go, rather than daily restarts.
# cfg009: flows from WDL, very little friction, nonhydrostatic.
# more reasonable dzmin_surface.
# cfg010: new grid (8.60), many tweaks, new bathy.
# cfg011: lowpass boundary conditions
run_dir=f"{args.dir}_{date_str}"
if not drv.SuntansModel.run_completed(run_dir):
grid_option='snubby'
if args.large:
def coamps_files(start, stop):
"""
Generate urls, filenames, and dates for
fetching or reading COAMPS data
Tries to pull the first 12 hours of runs, but if a run is known to be missing,
will pull later hours of an older run.
returns a generator, which yields for each time step of coamps output
{'wnd_utru':{'url=..., local=..., timestamp=...}, ...}
"""
dataset_name = "cencoos_4km"
# round to days
start = start.astype('M8[D]')
stop = stop.astype('M8[D]') + np.timedelta64(1, 'D')
# The timestamps we're trying for
target_hours = np.arange(start, stop, np.timedelta64(1, 'h'))
for hour in target_hours:
day_dt = utils.to_datetime(hour)
# Start time of the ideal run:
run_start0 = day_dt - datetime.timedelta(hours=day_dt.hour % 12)
# runs go for 48 hours, so we have a few chances to get the
# same output timestamp
for step_back in [0, 12, 24, 36]:
run_start = run_start0 - datetime.timedelta(hours=step_back)
# how many hours into this run is the target datetime?
hour_of_run = int(
round((day_dt - run_start).total_seconds() / 3600))
run_tag = "%04d%02d%02d%02d" % (run_start.year, run_start.month,
run_start.day, run_start.hour)
base_url = ("http://www.usgodae.org/pub/outgoing/fnmoc/models/"
"coamps/calif/cencoos/cencoos_4km/%04d/%s/") % (
run_start.year, run_tag)
recs = dict()
for field_name in ['wnd_utru', 'wnd_vtru', 'pres_msl']:
if field_name in ['wnd_utru', 'wnd_vtru']:
elev_code = 105 # 0001: surface? 0105: above surface 0100: pressure?
elev = 100
else:
# pressure at sea level
elev_code = 102
elev = 0
url_file = ("US058GMET-GR1dyn.COAMPS-CENCOOS_CENCOOS-n3-c1_"
"%03d"
"00F0NL"
"%s_%04d_%06d-000000%s") % (hour_of_run, run_tag,
elev_code, elev,
field_name)
output_fn = os.path.join(cache_path, dataset_name, url_file)
recs[field_name] = dict(url=base_url + url_file,
local=output_fn,
timestamp=hour)
if known_missing(recs):
continue
yield recs
break
else:
raise Exception("Couldn't find a run for date %s" %
day_dt.strftime('%Y-%m-%d %H:%M'))
def fill_and_flag(ds,fld,site,
lowpass_days=3*365,
shortgap_days=45 # okay to interpolate a little over a month?
):
"""
Update a single field for a single site in ds, by
extracting long-term trends, seasonal cycle, and
interpolating between these and measured data
"""
# first, create mapping from time index to absolute month
dts=utils.to_datetime(dns)
absmonth = [12*dt.year + (dt.month-1) for dt in dts]
absmonth = np.array(absmonth) - dts[0].year*12
month=absmonth%12
fld_in=ds[fld].sel(site=site)
orig_values=fld_in.values
fld_flag=ds[fld+'_flag'].sel(site=site)
prefilled=fld_flag.values & (FLAG_SEASONAL_TREND | FLAG_INTERP | FLAG_MEAN)
fld_in.values[prefilled]=np.nan # resets the work of this loop in case it's run multiple times
n_valid=np.sum(~fld_in.isnull())
if n_valid==0:
msg=" --SKIPPING--"
else:
msg=""
print(" field: %s %d/%d valid input points %s"%(fld,n_valid,len(fld_in),msg))
if n_valid==0:
return
# get the data into a monthly time series before trying to fit seasonal cycle
valid = np.isfinite(fld_in.values)
absmonth_mean=bin_mean(absmonth[valid],fld_in.values[valid])
month_mean=bin_mean(month[valid],fld_in.values[valid])
if np.sum(np.isfinite(month_mean)) < 12:
print("Insufficient data for seasonal trends - will fill with sample mean")
trend_and_season=np.nanmean(month_mean) * np.ones(len(dns))
t_and_s_flag=FLAG_MEAN
else:
# fit long-term trend and a stationary seasonal cycle
# this removes both the seasonal cycle and the long-term mean,
# leaving just the trend
trend_hf=fld_in.values - month_mean[month]
lp = filters.lowpass_fir(trend_hf,lowpass_days,nan_weight_threshold=0.01)
trend = utils.fill_invalid(lp)
# recombine with the long-term mean and monthly trend
# to get the fill values.
trend_and_season = trend + month_mean[month]
t_and_s_flag=FLAG_SEASONAL_TREND
# long gaps are mostly filled by trend and season
gaps=mark_gaps(dns,valid,shortgap_days,include_ends=True)
fld_in.values[gaps] = trend_and_season[gaps]
fld_flag.values[gaps] = t_and_s_flag
still_missing=np.isnan(fld_in.values)
fld_in.values[still_missing] = utils.fill_invalid(fld_in.values)[still_missing]
fld_flag.values[still_missing] = FLAG_INTERP
# Make sure all flows are nonnegative
negative=fld_in.values<0.0
fld_in.values[negative]=0.0
fld_flag.values[negative] |= FLAG_CLIPPED
if 0: # illustrative(?) plots
fig,ax=plt.subplots()
ax.plot(dns,orig_values,'m-o',label='Measured %s'%fld)
ax.plot(dns,fld_in,'k-',label='Final %s'%fld,zorder=5)
# ax.plot(dns,month_mean[month],'r-',label='Monthly Clim.')
# ax.plot(dns,trend_hf,'b-',label='Trend w/HF')
ax.plot(dns,trend,'g-',lw=3,label='Trend')
ax.plot(dns,trend_and_season,color='orange',label='Trend and season')
def write_t3d(da, suffix, feat_suffix, edge_depth, quantity, mdu):
"""
Write a 3D boundary condition for a feature from a vertical profile (likely
ROMS or HYCOM data)
- most of the time writing boundaries is here
- DFM details for rev52184:
the LAYERS line is silently truncated to 100 characters.
LAYER_TYPE=z assumes a coordinate of 0 at the bed, positive up
"""
run_base_dir = mdu.base_path
ref_date, t_start, t_stop = mdu.time_range()
# Luckily the ROMS output does not lose any surface cells - so we don't
# have to worry about a surface cell in roms_at_boundary going nan.
assert da.ndim in [2, 3]
# get the depth of the internal cell:
valid_depths = np.all(np.isfinite(da.values), axis=da.get_axis_num('time'))
valid_depths = valid_depths & (-da.depth.values > edge_depth)
# if this becomes a problem, may have to fill in backup value?
assert valid_depths[0], "No valid layers in Coastal model data"
valid_depth_idxs = np.nonzero(valid_depths)[0]
# ROMS values count from the surface, positive down.
# but DFM wants bottom-up.
# limit to valid depths, and reverse the order at the same time
da_sub = da.isel(depth=valid_depth_idxs[::-1])
max_line_length = 100 # limitation in DFM on the whole LAYERS line
# 7 is '_2.4567'
# -1 for minor bit of safety
max_layers = (max_line_length - len("LAYERS=")) // 7 - 1
# This should be the right numbers, but reverse order
sigma = (-edge_depth - da_sub.depth.values) / -edge_depth
# Force it to span the full water column
sigma[0] = min(0.0, sigma[0])
sigma[-1] = max(1.0, sigma[-1])
if len(sigma) > max_layers:
remapper = lambda y: np.interp(np.linspace(0, 1, max_layers),
np.linspace(0, 1, len(sigma)), y)
# Just because the use of remapper below is not compatible
# with vector quantities at this time.
assert da_sub.ndim - 1 == 1
else:
remapper = lambda y: y
sigma_str = " ".join(["%.4f" % s for s in remapper(sigma)])
# This line is truncated at 100 characters in DFM r52184.
layer_line = "LAYERS=%s" % sigma_str
assert len(layer_line) < max_line_length
elapsed_minutes = (da_sub.time.values - ref_date) / np.timedelta64(60, 's')
ref_date_str = utils.to_datetime(ref_date).strftime('%Y-%m-%d %H:%M:%S')
# assumes there are already node names
node_names = feat_suffix[2]
t3d_fns = [
os.path.join(run_base_dir, node_name + ".t3d")
for node_idx, node_name in enumerate(node_names)
]
assert da_sub.dims[0] == 'time' # for speed up of direct indexing
# Write the first, then copy it to the second node
with open(t3d_fns[0], 'wt') as fp:
fp.write("\n".join([
"LAYER_TYPE=sigma",
layer_line,
"VECTORMAX=%d" % (da_sub.ndim - 1), # default, but be explicit
"quant=%s" % quantity,
"quantity1=%s" % quantity, # why is this here?
"# start of data",
""
]))
for ti, t in enumerate(elapsed_minutes):
fp.write("TIME=%g minutes since %s\n" % (t, ref_date_str))
# Faster direct indexing:
# The ravel will interleave components - unclear if that's correct.
data = " ".join(
["%.3f" % v for v in remapper(da_sub.values[ti, :].ravel())])
fp.write(data)
fp.write("\n")
for t3d_fn in t3d_fns[1:]:
shutil.copyfile(t3d_fns[0], t3d_fn)
import argparse
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description='Download USGS transect data, format for extrapolation.')
# these will default to the period of the data.
parser.add_argument("-s", "--start", help="Starting date", required=True)
parser.add_argument("-e", "--end", help="Ending date", required=True)
parser.add_argument("-o",
"--output",
help="Path to CSV file for output",
default=None)
args = parser.parse_args()
start = np.datetime64(args.start)
end = np.datetime64(args.end)
if args.output is None:
args.output = "usgs_sfbay-%s-%s.csv" % (
utils.to_datetime(start).strftime('%Y%m%d'),
utils.to_datetime(end).strftime('%Y%m%d'))
log.info("Output file: %s" % args.output)
download_usgs(start=start, end=end, output=args.output)
# For direct testing
# download_usgs(start=np.datetime64("2012-09-01"),
# end= np.datetime64("2013-11-01"),
# output="test/test-usgs.csv")
f"{len(model_dirs)} model directories, spanning {models_start_time} to {models_end_time}"
)
for rel_time in rel_times:
end_time = rel_time + np.timedelta64(30, 'D')
assert rel_time >= models_start_time, "Set of model data starts after release time"
assert end_time < models_end_time, "Set of model data ends before end_time"
# turns out PTM crashes when making groups inactive.
# so have to do a single release period at a time. bummer.
# list of dicts for individual PTM calls.
calls = []
for rel_time in rel_times:
date_name = utils.to_datetime(rel_time).strftime('%Y%m%d')
for source in sources:
# 020b suffix to clarify hydro, and 'b' says we're doing per-source
run_dir = f"/opt2/sfb_ocean/ptm/all_source_020b/{source}/{date_name}"
if os.path.exists(run_dir):
# This will probably need to get a better test, for when a run
# failed.
print(f"Directory exists {run_dir}, will skip")
continue
call = dict(
model_dir=model_dirs[-1],
rel_times=[rel_time],
rel_duration=np.timedelta64(10, 'D'),
# group_duration=np.timedelta64(30,'D'),
end_time=rel_time + np.timedelta64(30, 'D'),
def dfmt(t):
return utils.to_datetime(t).strftime("%Y-%m-%d %H:%M:%S")
def loc_to_utc(t):
if utils.isnat(t): return t
naive = utils.to_datetime(t)
local_dt = local.localize(naive, is_dst=None)
utc_dt = local_dt.astimezone(pytz.utc)
return utils.to_dt64(utc_dt)
nc_fn=os.path.join( scalar_dir,scalar_name+".nc")
if os.path.exists(nc_fn):
print("%s data exists"%run_dir)
else:
print(run_dir)
scalar_map=dio.read_map( fn=os.path.join(run_dir,"sfb_dfm_v2.map"),
hyd=os.path.join(run_dir,"com-sfb_dfm_v2.hyd") )
scalar=scalar_map[scalar_name]
volumes=scalar_map['volume']
agg_scalar=np.zeros( (len(scalar_map.time),Nagg) )
for t_idx,t in enumerate(scalar_map.time):
print("Time: %s"%utils.to_datetime(t.values).strftime('%Y-%m-%d'))
# enforce ordering of dimensions for safety
scalar_value=scalar.isel(time=t_idx).transpose('layer','face').values
volume=volumes.isel(time=t_idx).transpose('layer','face').values
# need to filter out the -999 values first.
valid=(scalar_value!=-999)
num=(scalar_value*volume*valid).sum(axis=0)
den=(volume*valid).sum(axis=0)
#empty=(den<=0)
#num[empty]=0
#den[empty]=1.0
#scalar_2d= num/den
# And aggregate
def set_release_timing(self):
self.release_timing_names = [
"rel" + utils.to_datetime(rel_time).strftime('%Y%m%d')
for rel_time in self.rel_times
]
to_conc('NH3')
to_conc('PO4')
##
# The interpolation step - building off of synth_v02.py
fields=[s for s in ds.data_vars if not s.endswith('_flag')]
lowpass_days=3*365
shortgap_days=45 # okay to interpolate a little over a month?
# first, create mapping from time index to absolute month
dts=utils.to_datetime(dns)
absmonth = [12*dt.year + (dt.month-1) for dt in dts]
absmonth = np.array(absmonth) - dts[0].year*12
month=absmonth%12
for site in ds.site.values:
print("Site: %s"%site)
for fld in fields:
fld_in=ds[fld].sel(site=site)
orig_values=fld_in.values
fld_flag=ds[fld+'_flag'].sel(site=site)
prefilled=fld_flag.values & (FLAG_SEASONAL_TREND | FLAG_INTERP | FLAG_MEAN)
fld_in.values[prefilled]=np.nan # resets the work of this loop in case it's run multiple times
n_valid=np.sum( ~fld_in.isnull())
def query_usgs_sfbay(period_start, period_end, cache_dir=None, days_per_request='M'):
"""
Download (and locally cache) data from the monthly cruises of the USGS R/V Polaris
and R/V Peterson.
Returns data as pandas DataFrame.
"""
# Handle longer periods:
if days_per_request is not None:
logging.info("Will break that up into pieces")
dfs=[]
for interval_start,interval_end in periods(period_start,period_end,days_per_request):
df=query_usgs_sfbay(interval_start,interval_end,cache_dir=cache_dir,days_per_request=None)
if df is not None:
dfs.append(df)
return pd.concat(dfs)
params=[]
for column,text in usgs_sfbay_columns:
params.append( ('col',column) )
params += [('dstart','1990'),
('p11',''),
('p12',''),
('p21',''),
('p22',''),
('p31',''),
('p32','')]
comps=dict(type1='---',value1='',comp1='gt',
type2='---',value2='',comp2='gt',
type3='---',value3='',comp3='gt')
filter_count=0
for t,comp in [ (period_start,'ge'),
(period_end,'lt') ]:
if t is not None:
filter_count+=1
comps['type%d'%filter_count]='jdate'
comps['comp%d'%filter_count]=comp
comps['value%d'%filter_count]=str(utils.to_jdate(t))
for fld in ['type','value','comp']:
for comp_i in [1,2,3]:
fld_name=fld+str(comp_i)
params.append( (fld_name, comps[fld_name] ) )
params+= [ ('conj2','AND'),
('conj3','AND'),
('sort1','fulldate'),
('asc1','on'),
('sort2','stat'),
('asc2','on'),
('sort3','---'),
('asc3','on'),
('out','comma'),
('parm','on'),
('minrow','0'),
('maxrow','99999'),
('ftype','easy')
]
if cache_dir is not None:
fmt="%Y%m%d"
cache_file=os.path.join(cache_dir,'usgs_sfbay_%s_%s.csv'%(utils.to_datetime(period_start).strftime(fmt),
utils.to_datetime(period_end).strftime(fmt)))
else:
cache_file=None
def fetch():
logging.info("Fetch %s -- %s"%(period_start,period_end))
url="https://sfbay.wr.usgs.gov/cgi-bin/sfbay/dataquery/query16.pl"
result=requests.post(url,params)
text=result.text
soup = BeautifulSoup(text, 'html.parser')
data = soup.find('pre')
data1=data.get_text()
data2=re.sub(r'<!--[^>]*>','',data1) # Remove HTML comments
return data2
if cache_file is None:
data2=fetch()
else:
if not os.path.exists(cache_file):
data2=fetch()
with open(cache_file,'wt') as fp:
fp.write(data2)
else:
# print("Reading from cache")
logging.info("Cached %s -- %s"%(period_start,period_end))
with open(cache_file,'rt') as fp:
data2=fp.read()
df = pd.read_csv(StringIO(data2),skiprows=[1],parse_dates=["Date"] )
if len(df)==0:
return None
# get a real timestamp per station.
minutes=df.Time.values%100
hours=df.Time.values//100
time_of_day=(hours*3600+minutes*60).astype(np.int32) * np.timedelta64(1,'s')
df['time']=df['Date']+time_of_day
del df['Time']
# merge in lat/lon
lonlats=[station_number_to_lonlat(s) for s in df['Station Number']]
lonlats=np.array(lonlats)
df['longitude']=lonlats[:,0]
df['latitude']=lonlats[:,1]
return df
def query_usgs_sfbay(period_start, period_end, cache_dir=None):
params = []
for column, text in usgs_sfbay_columns:
params.append(('col', column))
params += [('dstart', '1990'), ('p11', ''), ('p12', ''), ('p21', ''),
('p22', ''), ('p31', ''), ('p32', '')]
comps = dict(type1='---',
value1='',
comp1='gt',
type2='---',
value2='',
comp2='gt',
type3='---',
value3='',
comp3='gt')
filter_count = 0
for t, comp in [(period_start, 'ge'), (period_end, 'lt')]:
if t is not None:
filter_count += 1
comps['type%d' % filter_count] = 'jdate'
comps['comp%d' % filter_count] = comp
comps['value%d' % filter_count] = str(utils.to_jdate(t))
for fld in ['type', 'value', 'comp']:
for comp_i in [1, 2, 3]:
fld_name = fld + str(comp_i)
params.append((fld_name, comps[fld_name]))
params += [('conj2', 'AND'), ('conj3', 'AND'), ('sort1', 'fulldate'),
('asc1', 'on'), ('sort2', 'stat'), ('asc2', 'on'),
('sort3', '---'), ('asc3', 'on'), ('out', 'comma'),
('parm', 'on'), ('minrow', '0'), ('maxrow', '99999'),
('ftype', 'easy')]
if cache_dir is not None:
fmt = "%Y%m%d"
cache_file = os.path.join(
cache_dir, 'usgs_sfbay_%s_%s.csv' %
(utils.to_datetime(period_start).strftime(fmt),
utils.to_datetime(period_end).strftime(fmt)))
else:
cache_file = None
def fetch():
url = "https://sfbay.wr.usgs.gov/cgi-bin/sfbay/dataquery/query16.pl"
result = requests.post(url, params)
text = result.text
soup = BeautifulSoup(text, 'html.parser')
data = soup.find('pre')
data1 = data.get_text()
data2 = re.sub(r'<!--[^>]*>', '', data1) # Remove HTML comments
return data2
if cache_file is None:
data2 = fetch()
else:
if not os.path.exists(cache_file):
data2 = fetch()
with open(cache_file, 'wt') as fp:
fp.write(data2)
else:
# print("Reading from cache")
with open(cache_file, 'rt') as fp:
data2 = fp.read()
df = pd.read_csv(StringIO(data2), skiprows=[1], parse_dates=["Date"])
return df
def samples_from_sfei_erddap(run_start, cache_dir=None):
"""
return [N,3] array of salinity data from SFEI moorings appropriate for
given date. Note that this may have no data, but will be returned
as a [0,3] array
This version fetches and caches data from SFEI's ERDDAP server
"""
if cache_dir is None:
cache_dir = common.cache_dir
dt_str = utils.to_datetime(run_start).strftime('%Y%m%d%H%M')
if cache_dir is not None:
my_cache_dir = os.path.join(cache_dir, 'enviz_erddap')
os.path.exists(my_cache_dir) or os.mkdir(my_cache_dir)
cache_fn = os.path.join(my_cache_dir, "temp_salt-%s.csv" % dt_str)
print("Cache fn: %s" % cache_fn)
else:
cache_fn = None
if cache_fn is not None and os.path.exists(cache_fn):
csv_data = cache_fn
else:
# Fetch data before/after run_start by this much
pad = np.timedelta64(30 * 60, 's')
fetch_period = [run_start - pad, run_start + pad]
fetch_strs = [
utils.to_datetime(p).strftime('%Y-%m-%dT%H:%M:00Z')
for p in fetch_period
]
# Because the table in ERDDAP is stored by sample, there is not guarantee that
# times are increasing. That makes access via opendap inefficient, so instead
# specify the query to ERDDAP more directly, and grab CSV for easier human
# readability
# choose dataset
base_url = "http://sfbaynutrients.sfei.org/erddap/tabledap/enviz_mirror.csv"
# choose fields to download
params = ",".join([
'stationcode', 'time', 'spcond_uS_cm', 'temp_C', 'stationname',
'latitude', 'longitude'
])
# And the time range
criteria = "time%%3E=%s&time%%3C=%s" % tuple(fetch_strs)
url = base_url + "?" + params + "&" + criteria
import requests
logging.info("Fetching SFEI data from %s" % url)
resp = requests.get(url)
if cache_fn is not None:
with open(cache_fn, 'wt') as fp:
fp.write(resp.content.decode())
csv_data = cache_fn
else:
csv_data = six.StringIO(resp.content.decode())
# 2nd row of file has units, which we ignore.
df = pd.read_csv(csv_data, skiprows=[1], parse_dates=['time'])
# Could get fancier and choose the closest in time reading, or
# interpolate. But this is not too bad, averaging over a total of
# 1 hour.
dfm = df.groupby('stationcode').mean()
# Get salinity from specific conductance
import seawater as sw
# specific conductance to mS/cm, and ratio to conductivityt at 35 psu, 15 degC.
# Note that mooring data comes in already adjusted to "specific conductance
# in uS/cm at 25 degC"
rt = dfm['spcond_uS_cm'].values / 1000. / sw.constants.c3515
dfm['salinity'] = sw.sals(rt, 25.0)
ll = np.c_[dfm.longitude.values, dfm.latitude.values]
xy = proj_utils.mapper('WGS84', 'EPSG:26910')(ll)
xys = np.c_[xy, dfm['salinity'].values]
valid = np.isfinite(xys[:, 2])
return xys[valid, :]
# Make sure run directory exists:
os.path.exists(run_base_dir) or os.makedirs(run_base_dir)
# clear any stale bc files:
for fn in [old_bc_fn]:
os.path.exists(fn) and os.unlink(fn)
## --------------------------------------------------------------------------------
# Edits to the template mdu:
#
mdu = dio.MDUFile('template.mdu')
if 1: # set dates
# RefDate can only be specified to day precision
mdu['time', 'RefDate'] = utils.to_datetime(ref_date).strftime('%Y%m%d')
mdu['time',
'Tunit'] = 'M' # minutes. kind of weird, but stick with what was used already
mdu['time', 'TStart'] = 0
mdu['time', 'TStop'] = int((run_stop - run_start) / np.timedelta64(1, 'm'))
mdu['geometry', 'LandBoundaryFile'] = os.path.join(rel_static_dir,
"deltabay.ldb")
mdu['geometry', 'Kmx'] = 10 # 10 layers
# update location of the boundary conditions
# this has the source/sinks which cannot be written in the new style file
mdu['external forcing', 'ExtForceFile'] = os.path.basename(old_bc_fn)
# Load the grid now -- it's used for clarifying some inputs, but
