def graph(site_name,field): default_interval=datetime.timedelta(hours=96) site=Site.by_name(site_name) last=site.last_reading() # stop=datetime.datetime.utcnow() stop=last.timestamp[0].to_pydatetime() start=stop - default_interval kwargs=dict(data_start_time=utils.to_unix(start), data_stop_time=utils.to_unix(stop)) kwargs['last_json']=last.iloc[0].to_json() kwargs['site_name']=site_name kwargs['field']=field # really ought to get this from the database if field.startswith('temp1'): unit='°F' elif field.startswith('press'): unit='Pa' elif field.startswith('pm'): unit='μg/m<sup>3</sup>' else: unit="n/a" kwargs['unit']=unit kwargs['columns']=[ col for col in last.columns if col!='timestamp'] return render_template('dygraph2.html',**kwargs)
def scan_ncs(self): self.nc_catalog=np.zeros( len(self.ncs), dtype=[('start_t','f8'), ('end_t','f8')] ) for nc_i,nc in enumerate(self.ncs): self.nc_catalog['start_t'][nc_i] = utils.to_unix(nc.time[0]) self.nc_catalog['end_t'][nc_i] = utils.to_unix(nc.time[-1])
def downsample(raw,period_secs): """ raw: pandas DataFrame, with a datetime64 time column """ timestamp=utils.to_unix(raw.time.values) periods=timestamp//period_secs #agg=raw.groupby(periods).agg( [np.min,np.mean,np.max] ) agg=raw.groupby(periods).agg( [np.mean] ) # drop multiindex by compounding the names new_cols=[] for col in agg.columns: if col[1]=='mean': new_col=col[0] else: new_col='_'.join(col) new_cols.append(new_col) agg.columns=new_cols unix_time=agg.index.values*period_secs agg['time']=np.datetime64("1970-01-01 00:00")+np.timedelta64(1,'s')*unix_time # reorder to get time first new_cols=['time'] + list(agg.columns[:-1]) agg=agg[new_cols] return agg
def graph2(): default_interval=datetime.timedelta(hours=96) site_name='rockridge' site=Site.by_name(site_name) last=site.last_reading() # stop=datetime.datetime.utcnow() stop=last.timestamp[0].to_pydatetime() start=stop - default_interval kwargs=dict(data_start_time=utils.to_unix(start), data_stop_time=utils.to_unix(stop)) kwargs['last_json']=last.iloc[0].to_json() kwargs['site_name']=site_name return render_template('dygraph2.html',**kwargs)
def parse_txts(txt_fns, pressure_range=[110e3, 225e3], name=None, beacon='auto', split_on_clock_change=True): """ Parse a collection of txt files, grab detections and optionally clock resync events. beacon: 'auto' set beacon tag id from most commonly received tag id. None: don't set a beacon tag id else: use the provided value as the beacon id. """ txt_fns = list(txt_fns) txt_fns.sort() dfs = [] for fn in utils.progress(txt_fns): df = parse_txt(fn) df['fn'] = fn dfs.append(df) df = pd.concat(dfs, sort=True) # sort columns to get alignment df = df.reset_index() # will use this for slicing later n_unknown = (df['type'] == 'unknown').sum() if n_unknown > 0: # So far, this is just corrupt lines. Might be able to # salvage the second part of corrupt lines, but it's # such a small number, and the second part is usually # just a NODE bootup msg anyway. log.warning("%d unknown line types in txt files" % n_unknown) # Do we need epoch? yes, it's used downstream epoch = utils.to_unix(df['time'].values) # It will get usec added, so be sure it's just the integer portion. assert np.all((epoch % 1.0)[np.isfinite(epoch)] == 0) df['epoch'] = epoch # Add microseconds to timestamps when t_usec is available sel = np.isfinite(df.t_usec.values) df.loc[sel, 'time'] = df.loc[ sel, 'time'] + (df.loc[sel, 't_usec'].values * 1e6).astype( np.int32) * np.timedelta64(1, 'us') df_det = df[df['type'] == 'DET'] # clean up time: bad_time = (df_det.time < np.datetime64('2018-01-01')) | ( df_det.time > np.datetime64('2022-01-01')) df2 = df_det[~bad_time].copy() # clean up temperature: df2.loc[df2.temp < -5, 'temp'] = np.nan df2.loc[df2.temp > 35, 'temp'] = np.nan # clean up pressure if pressure_range is not None: df2.loc[df2.pressure < pressure_range[0], 'pressure'] = np.nan df2.loc[df2.pressure > pressure_range[1], 'pressure'] = np.nan # trim to first/last valid pressure valid_idx = np.nonzero(np.isfinite(df2.pressure.values))[0] df3 = df2.iloc[valid_idx[0]:valid_idx[-1] + 1, :].copy() df3['tag'] = [s.strip() for s in df3.tag.values] # narrow that to the fields we actually care about: fields = [ 'rx_serial', 'tag', 'time', 't_usec', 'epoch', 'corrQ', 'nbwQ', 'pressure', 'temp', 'datetime_str', 'fn' ] ds = xr.Dataset.from_dataframe(df3.loc[:, fields]) ds['usec'] = ds['t_usec'] ds['cf2_filename'] = None if beacon == 'auto': beacon_id = df3.groupby('tag').size().sort_values().index[-1] ds['beacon_id'] = beacon_id ds['beacon_id'].attrs['source'] = 'received tags' print("auto_beacon: beacon_id inferred as ", beacon_id) # import pdb # pdb.set_trace() elif beacon is not None: ds['beacon_id'] = beacon ds['beacon_id'].attrs['source'] = 'Specified to parse_txts' ds.attrs['pressure_range'] = pressure_range if name is not None: ds['name'] = (), name if split_on_clock_change: # dice_by_clock_resets got crazy complicated. Rather than # re-living that experience, try something simple here, # but know that we may have to come back and refactor this # with dice_by_clock_resets. sync_sel = df['sync_status'].values == 1.0 sync_idxs = df.index.values[ sync_sel] # of course these won't actually be in df3! nonmono_sel = np.diff(df3.time.values) < np.timedelta64(0) # Should mean that each item is the first index of a new chunk. nonmono_idxs = df3.index.values[1:][nonmono_sel] all_breaks = np.unique(np.concatenate((sync_idxs, nonmono_idxs))) all_breaks = np.r_[0, all_breaks, len(df)] # as indices into ds.index ds_breaks = np.searchsorted(ds.index.values, all_breaks) diced = [] for start, stop in zip(ds_breaks[:-1], ds_breaks[1:]): if stop > start: diced.append(ds.isel(index=slice(start, stop))) else: # often will have a slice that's empty. pass return diced else: return ds
def write_inlet_morpho(self): apply_to_nodes=True # patched schism -- applies dumping as dz at nodes when index is negative ds=self.prep_qcm_data() # Too lazy to push this into shapefile morph_poly=wkt.loads(self.morph_wkt) # Update dumping every 15 min dump_dt_s=900 sim_seconds=(self.run_stop-self.run_start)/np.timedelta64(1,'s') # start at t=0 but we won't output dumping at t=0 dump_t=np.arange(0,sim_seconds+dump_dt_s,dump_dt_s) # target elevation z_thalweg=np.interp( utils.to_unix(self.run_start)+dump_t, utils.to_unix(ds.time.values),ds['z_thalweg'].values) if apply_to_nodes: self.morph_nodes=self.grid.select_nodes_intersecting(morph_poly,as_type='indices') #nodes=np.unique( np.concatenate([self.grid.cell_to_nodes(c) for c in elts])) nodes=self.morph_nodes area=np.ones(len(nodes)) # direct elevation, so no area adjustment elts1=-(nodes+1) # negate and to 1-based z_bed=self.grid.nodes['node_z_bed'][nodes] else: self.morph_elts=self.grid.select_cells_intersecting(morph_poly,as_type='indices') elts=self.morph_elts area=self.grid.cells_area()[elts] # A little tricky to figure out ground elevation -- bathy is at nodes. # take average to estimate element bathy z_cells=self.grid.interp_node_to_cell(self.grid.nodes['node_z_bed']) z_bed=z_cells[elts] elts1=1+elts # 1-based # For now, the same z_target for all elements z_target=np.zeros( (len(dump_t),len(elts1)), np.float64) z_target[:,:]=np.maximum( z_thalweg[:,None], # broadcast the same target over all elements z_bed[None,:] ) # broadcast bathy over all time z_target=np.round_(z_target, 3) # Don't care about sub-mm variation. just makes the files bigger. z_target[0,:]=z_bed # initial condition dz_target=np.diff(z_target,axis=0) with open(os.path.join(self.run_dir,'sed_dump.in'),'wt') as fp: fp.write("from QCM, update elevation every %s s\n"%dump_dt_s) # Possible bug in sediment.F90 when the simulation starts on or before # the time of the first dump record. # Here skip first dump_t, which is the initial condition, such that # the dumping at time t reflects dz calculated over the preceding time # step of dump_dt_s for ti,t in enumerate(dump_t[1:]): dzs=dz_target[ti,:] active=(dzs!=0.0) if not np.any(active): continue active_elts1=elts1[active] fp.write('%g %d\n'%(t,len(active_elts1))) # t_dump, ne_dump dVs=(dzs*area)[active] for elt1,dV in zip(active_elts1,dVs): # read(18,*)(ie_dump(l),vol_dump(l),l=1,ne_dump) # to 1-based fp.write('%d %.4f\n'%(elt1,dV))
# Don't get too big for our britches, just stick a second node 50m east # if the incoming data is a point if 1: #-- Write a PLI file pnts=np.array( [[stn_ds.utm_x,stn_ds.utm_y], [stn_ds.utm_x + 50.0,stn_ds.utm_y]] ) pli_data=[ (src_name,pnts) ] pli_fn=opj(out_dir,"%s.pli"%src_name) dio.write_pli(pli_fn,pli_data) pli_files.append(pli_fn) if 1: #-- Write a BC file df=stn_ds.to_dataframe().reset_index() df['unix_time']=utils.to_unix(df.time.values) bc_fn=opj(out_dir,"%s.bc"%src_name) with open(bc_fn,'wt') as fp: # I think the 0001 has to be there, as it is used to # specify different values at different nodes of the pli # seems better to assume that incoming data is a daily average, # and keep it constant across the day # block-from means that the timestamp of a datum marks the beginning # of a period, and the value is held constant until the next timestamp # how about unix epoch for time units? fp.write("[forcing]\n") # This Name needs to match the name in the pli fp.write("Name = %s_0001\n"%src_name) fp.write("Function = timeseries\n") fp.write("Time-interpolation = block-from\n") # or linear, block-to
def set_current_nc(self, nc_i): self.current_nc_idx = nc_i self.current_nc = self.ncs[self.current_nc_idx] self.nc_t_unix = utils.to_unix(self.current_nc.time.values) self.nc_time_i = -999
def df_post(df): """ add cms, and unix time, in place. """ df['flow_cms'] = 0.028316847 * df.flow_cfs df['unix_time'] = utils.to_unix(df.date.values)
ds2=ds.rename({'ucxa':'cell_east_velocity', 'ucya':'cell_north_velocity'}) ## ptm_fwd=Streamlines(ds2,grid=g) ptm_rev=Streamlines(ds2,grid=g,reverse=True) ## # debugging a crash parts=xyz_input[::30,:2][518:519] ptm=Streamlines(ds2,grid=g,record_dense=True) ptm.add_particles(x=parts) t0=np.datetime64("2000-01-01 00:00") t_out=utils.to_unix(t0+np.timedelta64(10,'s')*np.arange(31)) ptm.set_time(t_out[0]) ptm.integrate(t_out) # 946685100.0 locs=np.array( [coord for coord,time in ptm.dense] ) ## plt.figure(1).clf() ax=plt.gca() ptm.g.plot_edges(clip=clip,color='k',lw=0.4,ax=ax) ax.plot(locs[:,0,0],locs[:,0,1],'g-o') ##
tvalid = utils.fill_invalid(times, axis=-1) tvalid = utils.fill_invalid(tvalid, axis=1) tvalid = utils.fill_invalid(tvalid, axis=0) times = tvalid ### Load in his files, define variables, adjust times to same reference time as USGS data temp = True his = nc.MFDataset(path + hisfile) xcoor = his.variables["station_x_coordinate"][:] ycoor = his.variables["station_y_coordinate"][:] # get reference time from the mdu: t_ref, t_start, t_stop = mdu.time_range() dref = utils.to_unix(t_ref) # dt.datetime.strptime(emdu['time','refdate'] dtz = -8 * 60 * 60 # adjust UTC to PST mtimes = his.variables["time"][:] + dref + dtz msalt = his.variables["salinity"][:] if temp == True: mtemp = his.variables["temperature"][:] mdepth = his.variables["zcoordinate_c"][:] if xcoor.ndim == 2: # some DFM runs add time to coordinates xcoor = xcoor[0, :] ycoor = ycoor[0, :] mll = utm_to_ll(np.c_[xcoor, ycoor]).T
ax_temp.legend(loc='upper left') fig.tight_layout() fig.savefig(os.path.join(fig_dir, 'timeseries-arrival_flow_turb.png')) ## # And presence, swimming spatially -- look at screen_track_with_hydro # all_segs= # Seem to have lost my turbidity plots... # Add turbidity, flow, doy to df_start df_start['turb'] = np.interp(df_start.t_mid, utils.to_unix(msd_turb.time.values), msd_turb['turb_lp'].values) df_start['flow_m3s'] = np.interp(df_start.t_mid, utils.to_unix(msd_flow.time.values), msd_flow.flow_m3s) df_start['velo_ms'] = np.interp(df_start.t_mid, utils.to_unix(msd_velo.time.values), msd_velo.velocity_ms) df_start['time_mid'] = utils.unix_to_dt64(df_start.t_mid) df_start['doy'] = df_start.time_mid.dt.dayofyear # For comparison below, get a velocity calculated (roughly) for the analysis polygon # This is similar to velo_ms, but has a bit more range import transport_analysis as ta reach_velo = ta.calc_reach_velo(df_start.time_mid.values)
nc_tri['depth']=('face',),new_depth nc_tri['surface']=('face',),ds2['s1'].values[mixed_to_tri] # kludge - hardwire the sign of edgedepth, positive down nc_tri['edgedepth']=('edge',),-(ds2['NetNode_z'].values[ gtri.edges['nodes'] ].mean(axis=1)) ## ptm=StreamlinesKGS(nc_tri,grid=gtri) # even that init is slow # probably a lot of this would be faster by using DFM's unorm. parts=xyz_input[::100,:2] ptm.add_particles(x=parts) t0=np.datetime64("2000-01-01 00:00") t_out=utils.to_unix(t0+np.timedelta64(30,'s')*np.arange(5)) ptm.set_time(t_out[0]) ## ptm.integrate(t_out) ## # 946685100.0 locs=np.array( [coord for coord,time in ptm.output] ) ## plt.figure(1).clf() ax=plt.gca()