def test_provisional(): usgs_a8_site = "372512121585801" ds = usgs_nwis.nwis_dataset(usgs_a8_site, np.datetime64("2015-12-10"), np.datetime64("2015-12-20"), products=[72178])
def nudge_by_gage(ds, usgs_station, station, decorr_days, period_start=None, period_end=None): # This slicing may be stopping one sample shy, shouldn't be a problem. if period_start is None: period_start = ds.time.values[0] if period_end is None: period_end = ds.time.values[-1] usgs_gage = usgs_nwis.nwis_dataset(usgs_station, products=[60], start_date=period_start, end_date=period_end, days_per_request='M', cache_dir=common.cache_dir) # Downsample to daily df = usgs_gage['stream_flow_mean_daily'].to_dataframe() df_daily = df.resample('D').mean() # Get the subset of BAHM data which overlaps this gage data time_slc = slice(np.searchsorted(ds.time, df_daily.index.values[0]), 1 + np.searchsorted(ds.time, df_daily.index.values[-1])) bahm_subset = ds.sel(station=station).isel(time=time_slc) assert len(bahm_subset.time) == len( df_daily), "Maybe BAHM data doesn't cover entire period" errors = bahm_subset.flow_cfs - df_daily.stream_flow_mean_daily # Easiest: interpolate errors over nans, apply to bahm data array. # the decorrelation time is tricky, though. # Specify a decorrelation time scale then relax from error to zero # over that period valid = np.isfinite(errors.values) errors_interp = np.interp(utils.to_dnum(ds.time), utils.to_dnum(df_daily.index[valid]), errors[valid]) all_valid = np.zeros(len(ds.time), 'f8') all_valid[time_slc] = 1 * valid weights = (2 * filters.lowpass_fir(all_valid, decorr_days)).clip(0, 1) weighted_errors = weights * errors_interp # Does this work? subset = dict(station=station) cfs_vals = ds.flow_cfs.loc[subset] - weighted_errors ds.flow_cfs.loc[subset] = cfs_vals.clip(0, np.inf) ds.flow_cms.loc[subset] = 0.028316847 * ds.flow_cfs.loc[subset] # user feedback cfs_shifts = weighted_errors[time_slc] print("Nudge: %s => %s, shift in CFS: %.2f +- %.2f" % (usgs_station, station, np.mean(cfs_shifts), np.std(cfs_shifts)))
def test_basic(): # This requires internet access! ds=usgs_nwis.nwis_dataset(station="11337190", start_date=np.datetime64('2012-08-01'), end_date =np.datetime64('2012-10-01'), products=[60, # "Discharge, cubic feet per second" 10], # "Temperature, water, degrees Celsius" days_per_request=30)
def test_missing(): station = "11162765" t_start = np.datetime64('2016-10-01') t_stop = np.datetime64('2016-12-01') # This period has some missing data identified by '***' which # caused problems in older versions of rdb.py ds = usgs_nwis.nwis_dataset(station, t_start, t_stop, products=[95, 90860], days_per_request=20)
def usgs_salinity_time_series(station): # A little tricky - there are two elevations, which have the same parameter # code of 95 for specific conductance, but ts_id's of 14739 and 14741. # requesting the parameter once does return an RDB with both in there. time_labels = [ utils.to_datetime(t).strftime('%Y%m%d') for t in [t_start, t_stop] ] cache_fn = "usgs%s-%s_%s-salinity.nc" % (station, time_labels[0], time_labels[1]) if not os.path.exists(cache_fn): # 95: specific conductance # 90860: salinity ds = usgs_nwis.nwis_dataset(station, t_start, t_stop, products=[95, 90860], days_per_request=20) usgs_nwis.add_salinity(ds) ds.to_netcdf(cache_fn) ds.close() ## ds = xr.open_dataset(cache_fn) ds.attrs['lon'] = station_locs[station]['lon'] ds.attrs['lat'] = station_locs[station]['lat'] ds.salinity.attrs['elev_mab'] = station_locs[station]['elev_mab'][0] if 'salinity_01' in ds: ds.salinity_01.attrs['elev_mab'] = station_locs[station]['elev_mab'][1] xy = ll2utm([ds.attrs['lon'], ds.attrs['lat']]) ds.attrs['x'] = xy[0] ds.attrs['y'] = xy[1] return ds
def test_caching(): station = "11162765" t_start = np.datetime64('2016-10-01') t_stop = np.datetime64('2016-12-01') cache_dir = 'tmp_cache' if os.path.exists(cache_dir): # Start clean shutil.rmtree(cache_dir) os.mkdir(cache_dir) timings = [] for trial in [0, 1]: t0 = time.time() ds = usgs_nwis.nwis_dataset(station, t_start, t_stop, products=[95, 90860], days_per_request='10D', cache_dir=cache_dir) timings.append(time.time() - t0) assert timings[0] > 5 * timings[1]
srv=11455420, # will have to get nudged a bit # ryi=11455350, # no data in this particular period hwb=11455165, # miner slough at hwy 84 dws=11455335, # deep water ship channel, near RV ame=11446980, # American River, may have to get nudged ver=11425500, cli=11455315, # Cache Slough at S. Liberty Island cah=11455280, # Cache Slough near Hastings Tract ula=11455268, # Ulatis near Elmira ) stations=[] for stn_name in usgs_station_codes: stn_code=usgs_station_codes[stn_name] ds=usgs_nwis.nwis_dataset(stn_code,run_start,run_stop,products=[10], cache_dir=cache_dir,cache_only=True) if ds is None: print("No data for %s -- skip."%stn_code) continue meta=usgs_nwis.station_metadata(stn_code,cache_dir=cache_dir) ds['latitude']=(),meta['lat'] ds['longitude']=(),meta['lon'] ll=np.array( [ds.longitude.values,ds.latitude.values]) ds['ll']=('xy',),ll xy=ll2utm(ll) ds['xy']=('xy',),xy ds.attrs['name']=stn_name stations.append(ds) # Based on the locations, the American River Station is maybe too far # away, and since the model doesn't have American River flows, it
def add_delta_inflow(mdu, rel_bc_dir, static_dir, grid, dredge_depth, all_flows_unit=False, temp_jersey=True, temp_rio=True): """ Fetch river USGS river flows, add to FlowFM_bnd.ext: Per Silvia's Thesis: Jersey: Discharge boundary affected by tides, discharge and temperature taken from USGS 11337190 SAN JOAQUIN R A JERSEY POINT, 0 salinity (Note that Dutch Slough should probably be added in here) Rio Vista: 11455420 SACRAMENTO A RIO VISTA, temperature from DWR station RIV. 0 salinity. run_base_dir: location of the DFM inputs run_start,run_stop: target period for therun statiC_dir: path to static assets, specifically Jersey.pli and RioVista.pli grid: UnstructuredGrid instance, to be modified at inflow locations old_bc_fn: path to old-style boundary forcing file all_flows_unit: if True, override all flows to be 1 m3 s-1 for model diagnostics """ # get run directory and time and forcing file info run_base_dir = mdu.base_path ref_date, run_start, run_stop = mdu.time_range() old_bc_fn = mdu.filepath(["external forcing", "ExtForceFile"]) pad = np.timedelta64(3, 'D') if 1: # Cache the original data from USGS, then clean it and write to DFM format jersey_raw_fn = os.path.join(run_base_dir, rel_bc_dir, 'jersey-raw.nc') if not os.path.exists(jersey_raw_fn): if temp_jersey == True: jersey_raw = usgs_nwis.nwis_dataset( station="11337190", start_date=run_start - pad, end_date=run_stop + pad, products=[ 60, # "Discharge, cubic feet per second" 10 ], # "Temperature, water, degrees Celsius" days_per_request=30) jersey_raw.to_netcdf(jersey_raw_fn, engine='scipy') if temp_jersey == False: jersey_raw = usgs_nwis.nwis_dataset( station="11337190", start_date=run_start - pad, end_date=run_stop + pad, products=[60], # "Discharge, cubic feet per second" days_per_request=30) jersey_raw.to_netcdf(jersey_raw_fn, engine='scipy') rio_vista_raw_fn = os.path.join(run_base_dir, rel_bc_dir, 'rio_vista-raw.nc') if not os.path.exists(rio_vista_raw_fn): if temp_rio == True: rio_vista_raw = usgs_nwis.nwis_dataset( station="11455420", start_date=run_start - pad, end_date=run_stop + pad, products=[ 60, # "Discharge, cubic feet per second" 10 ], # "Temperature, water, degrees Celsius" days_per_request=30) rio_vista_raw.to_netcdf(rio_vista_raw_fn, engine='scipy') if temp_rio == False: rio_vista_raw = usgs_nwis.nwis_dataset( station="11455420", start_date=run_start - pad, end_date=run_stop + pad, products=[60], # "Discharge, cubic feet per second" days_per_request=30) rio_vista_raw.to_netcdf(rio_vista_raw_fn, engine='scipy') if 1: # Clean and write it all out jersey_raw = xr.open_dataset(jersey_raw_fn) rio_vista_raw = xr.open_dataset(rio_vista_raw_fn) temp_logical = [temp_jersey, temp_rio] i = 0 for src_name, source in [('Jersey', jersey_raw), ('RioVista', rio_vista_raw)]: src_feat = dio.read_pli( os.path.join(static_dir, '%s.pli' % src_name))[0] dredge_grid.dredge_boundary(grid, src_feat[1], dredge_depth) if temp_logical[i] == True: # Add stanzas to FlowFMold_bnd.ext: for quant, suffix in [('dischargebnd', '_flow'), ('salinitybnd', '_salt'), ('temperaturebnd', '_temp')]: with open(old_bc_fn, 'at') as fp: lines = [ "QUANTITY=%s" % quant, "FILENAME=%s/%s%s.pli" % (rel_bc_dir, src_name, suffix), "FILETYPE=9", "METHOD=3", "OPERAND=O", "" ] fp.write("\n".join(lines)) feat_suffix = dio.add_suffix_to_feature(src_feat, suffix) dio.write_pli( os.path.join(run_base_dir, rel_bc_dir, '%s%s.pli' % (src_name, suffix)), [feat_suffix]) # Write the data: if quant == 'dischargebnd': da = source.stream_flow_mean_daily da2 = utils.fill_tidal_data(da) if all_flows_unit: da2.values[:] = 1.0 else: # convert ft3/s to m3/s da2.values[:] *= 0.028316847 elif quant == 'salinitybnd': da2 = source.stream_flow_mean_daily.copy(deep=True) da2.values[:] = 0.0 elif quant == 'temperaturebnd': da = source.temperature_water da2 = utils.fill_tidal_data( da) # maybe safer to just interpolate? if all_flows_unit: da2.values[:] = 20.0 df = da2.to_dataframe().reset_index() df['elapsed_minutes'] = ( df.time.values - ref_date) / np.timedelta64(60, 's') columns = ['elapsed_minutes', da2.name] if len(feat_suffix) == 3: node_names = feat_suffix[2] else: node_names = [""] * len(feat_suffix[1]) for node_idx, node_name in enumerate(node_names): # if no node names are known, create the default name of <feature name>_0001 if not node_name: node_name = "%s%s_%04d" % (src_name, suffix, 1 + node_idx) tim_fn = os.path.join(run_base_dir, rel_bc_dir, node_name + ".tim") df.to_csv(tim_fn, sep=' ', index=False, header=False, columns=columns) if temp_logical[i] == False: # Add stanzas to FlowFMold_bnd.ext: for quant, suffix in [('dischargebnd', '_flow'), ('salinitybnd', '_salt')]: with open(old_bc_fn, 'at') as fp: lines = [ "QUANTITY=%s" % quant, "FILENAME=%s/%s%s.pli" % (rel_bc_dir, src_name, suffix), "FILETYPE=9", "METHOD=3", "OPERAND=O", "" ] fp.write("\n".join(lines)) feat_suffix = dio.add_suffix_to_feature(src_feat, suffix) dio.write_pli( os.path.join(run_base_dir, rel_bc_dir, '%s%s.pli' % (src_name, suffix)), [feat_suffix]) # Write the data: if quant == 'dischargebnd': da = source.stream_flow_mean_daily da2 = utils.fill_tidal_data(da) if all_flows_unit: da2.values[:] = 1.0 else: # convert ft3/s to m3/s da2.values[:] *= 0.028316847 elif quant == 'salinitybnd': da2 = source.stream_flow_mean_daily.copy(deep=True) da2.values[:] = 0.0 df = da2.to_dataframe().reset_index() df['elapsed_minutes'] = ( df.time.values - ref_date) / np.timedelta64(60, 's') columns = ['elapsed_minutes', da2.name] if len(feat_suffix) == 3: node_names = feat_suffix[2] else: node_names = [""] * len(feat_suffix[1]) for node_idx, node_name in enumerate(node_names): # if no node names are known, create the default name of <feature name>_0001 if not node_name: node_name = "%s%s_%04d" % (src_name, suffix, 1 + node_idx) tim_fn = os.path.join(run_base_dir, rel_bc_dir, node_name + ".tim") df.to_csv(tim_fn, sep=' ', index=False, header=False, columns=columns) i += 1
from stompy.io.local import usgs_nwis from stompy import utils import os import matplotlib.pyplot as plt ## cache_dir = "cache" if not os.path.exists(cache_dir): os.makedirs(cache_dir) ds = usgs_nwis.nwis_dataset(11455420, np.datetime64("2019-08-01"), np.datetime64("2019-09-01"), products=[65], cache_dir=cache_dir) ds_long = usgs_nwis.nwis_dataset(11455420, np.datetime64("2019-07-01"), np.datetime64("2019-10-01"), products=[65], cache_dir=cache_dir) ## # Try basic tidal filling: height_filled = utils.fill_tidal_data(ds.height_gage) height_filled_long = utils.fill_tidal_data(ds_long.height_gage)
ges_april=pd.read_csv( ('/home/rusty/mirrors/Ed/Sync/UCD/Projects/CDFW_Arc/dflowfm/' '27-comp_bedlevtyp_2_3_4/stations/wsel/' 'GES_STAGE_april2014.csv'), parse_dates=['Time']) sel=(ges_mwtract.Time>=ges_april.Time.iloc[0])&(ges_mwtract.Time<=ges_april.Time.iloc[-1]) ## from stompy.io.local import usgs_nwis cache_dir='cache' os.path.exists(cache_dir) or os.mkdir(cache_dir) ges_ds=usgs_nwis.nwis_dataset(station="11447905", start_date=np.datetime64("2014-04-01"), end_date=np.datetime64("2014-05-01"), cache_dir=cache_dir, products=[60,65], # discharge and stage days_per_request='M') # that always returns UTC, so add a PST time: ges_ds['time_pst']=('time',), ges_ds.time + np.timedelta64(-8,'h') ## import matplotlib.pyplot as plt plt.ion() plt.figure(12).clf() # These are all in agreement now. plt.plot( ges_april.Time, ges_april.Stage,label='april') plt.plot( ges_mwtract.Time[sel], ges_mwtract.Stage[sel],label='MWT')
# Get a sense of how to fabricate Decker Island tides when only Rio Vista # data is available. # choosing not to do this -- better to truncate the domain at Rio Vista. import numpy as np from stompy.io.local import usgs_nwis from stompy import utils import matplotlib.pyplot as plt ## period = [np.datetime64("2016-01-01"), np.datetime64("2019-01-01")] decker = usgs_nwis.nwis_dataset(station=11455478, start_date=period[0], end_date=period[1], products=[60, 65], cache_dir='cache') riovista = usgs_nwis.nwis_dataset(station=11455420, start_date=period[0], end_date=period[1], products=[60, 65], cache_dir='cache') ## ## from stompy import filters # separate into tidal, subtidal for ds in [decker, riovista]:
axs[0].plot(adcp.time, adcp.u_davg, label='U davg') axs[1].plot(adcp.time, adcp.v_davg, label='V davg') axs[2].plot(ds.time, ds.stream_flow_mean_daily, label='Rio new Raw') #axs[2].plot(rio.time + np.timedelta64(8,'h'), # -rio.stream_flow_mean_daily, # label='Rio Flip/shift') axs[0].legend() axs[1].legend() axs[2].legend() axs[2].axis((735104.96660250274, 735107.16195219487, -126058.97073039225, 127174.6378024802)) ## # Okay - run_start = adcp.time[0].astype('M8[D]') run_stop = run_start + np.timedelta64(10, 'D') from stompy.io.local import usgs_nwis import pdb ds = usgs_nwis.nwis_dataset(station="11455420", start_date=run_start, end_date=run_stop, products=[60], days_per_request=30)
"381934121403201"), # Liberty Island, Wildlands, Up Marsh # no physical data until 2015-07: ("LIB", "11455315"), # Cache Slough A S Liberty Island Nr Rio Vista CA ("TSL", "11337080"), # Threemile slough near Rio Vista ("SDI", "11455478"), # Sac River at Decker Island ]: stage_fn = '%s-stage.csv' % usgs_name flow_fn = '%s-flow.csv' % usgs_name if os.path.exists(stage_fn) and os.path.exists(flow_fn): continue # or force. print("Downloading %s" % usgs_name) ds = usgs_nwis.nwis_dataset( usgs_station, download_period[0], download_period[1], [60, 65], # Discharge and Stage days_per_request='M', cache_dir=cache_dir) # nwis_dataset() returns UTC data. Convert to PST: ds['time'] = ds.time - np.timedelta64(8, 'h') # Match the names up with existing csv files: ds = ds.rename({'time': 'Time'}) if 'stream_flow_mean_daily' in ds: ds = ds.rename({'stream_flow_mean_daily': 'Flow'}) if 'height_gage' in ds: ds = ds.rename({'height_gage': 'Stage'}) df = ds.to_dataframe() if 'Stage' in df:
pd.read_csv('forcing-data/Barker_Pumping_Plant.csv', parse_dates=['time'])) barker = barker.set_coords('time') model.add_FlowBC(name='Barker_Pumping_Plant', Q=barker['Q']) rio_vista = model.read_bc('forcing-data/WaterLevel.bc')['SRV_0001'] model.add_StageBC(name='SRV', z=rio_vista['waterlevelbnd']) Qshared = model.read_bc('forcing-data/Discharge.bc') model.add_FlowBC(name='Georgiana', Q=Qshared['Georgiana_0001']['dischargebnd']) # DXC was closed during this period. # model.add_FlowBC(name='DXC',Q=Qshared['DXC_0001']['dischargebnd']) if 1: # freeport flows, lowpass ds_fpx = usgs_nwis.nwis_dataset( station=11447650, start_date=model.run_start - np.timedelta64(5, 'D'), end_date=model.run_stop + np.timedelta64(5, 'D'), products=[60, 65], cache_dir="cache") ds_fpx.time.values -= np.timedelta64(8, 'h') # convert UTC to PST. # original data had flows shifted 14.5h, try just 2h? it's possible that they really should # be lagged, not shifted back in time, since the signal is mostly tidal and we're talking # about propagation of the tides. ds_fpx.time.values -= np.timedelta64(int(2. * 3600), 's') ds_fpx.stream_flow_mean_daily.values *= 0.028316847 # cfs => m3/s # that's 15 minute data. flow = ds_fpx.stream_flow_mean_daily flow.values[:] = filters.lowpass_godin(flow.values, utils.to_dnum(flow.time)) model.add_FlowBC(name="SacramentoRiver", Q=flow) # Try file pass through for forcing data: