def fun(i):
num=ctdnum[i]
print(num)
try:
mod=load_ctd_mod('{}ctd_timeseries_{}.txt'.format(modpath,num))
obs=load_ctd_obs('{}{}_ctd.dat'.format(obspath,num))
other['num']=num
other['tidx']=np.argmin(np.fabs(mod['arrays']['time'][0,:]-obs['time']))
#for i in range(mod['arrays']['depth'].shape[1]):
Tmod=ipt.interp1d(-1*mod['arrays']['depth'][:,other['tidx']],mod['arrays']['temperature'][:,other['tidx']],obs['Depth'])
Smod=ipt.interp1d(-1*mod['arrays']['depth'][:,other['tidx']],mod['arrays']['salinity'][:,other['tidx']],obs['Depth'])
cTmod, cTobs = remove_common_nan(Tmod,obs['Temp'])
cSmod, cSobs = remove_common_nan(Smod,obs['Salinity'])
Tstats['{}'.format(num)]=residual_stats(cTmod, cTobs)
Sstats['{}'.format(num)]=residual_stats(cSmod, cSobs)
other['filename']='{}ctd_timeseries_{}.png'.format(savepath,num)
plot_tsmap(mod,obs,other)
except:
print('Pass on {}'.format(num))
pass
def interp_common(time1,data1,time2,data2,dt=60):
# tmin is the biggest small number
# tmax is the smallest big number
tmin=np.max([time1[0],time2[0]])
tmax=np.min([time1[-1],time2[-1]])
time=np.arange(tmin,tmax+dt/(24*60*60.0),dt/(24*60*60.0))
data1n=ipt.interp1d(time1,data1,time)
data2n=ipt.interp1d(time2,data2,time)
#return time and data
return time,data1n,data2n
def interp_common(time1, data1, time2, data2, dt=60):
# tmin is the biggest small number
# tmax is the smallest big number
tmin = np.max([time1[0], time2[0]])
tmax = np.min([time1[-1], time2[-1]])
time = np.arange(tmin, tmax + dt / (24 * 60 * 60.0), dt / (24 * 60 * 60.0))
data1n = ipt.interp1d(time1, data1, time)
data2n = ipt.interp1d(time2, data2, time)
#return time and data
return time, data1n, data2n
def interp_clean_common(time1,
data1,
time2,
data2,
filter_max=1000.0,
filter_min=-1000.0):
#copy and filter dataset 1
data11 = copy.copy(data1)
data11[data1 >= filter_max] = np.nan
data11[data1 <= filter_min] = np.nan
#copy and filter dataset 2
data22 = copy.copy(data2)
data22[data2 >= filter_max] = np.nan
data22[data2 <= filter_min] = np.nan
# interp filtered data2 to dataset1 times
#print(time2.shape,data22.shape,time1.shape)
data21 = ipt.interp1d(time2, data22, time1)
#idx to remove all nans from either dataset
bidx = np.isnan(data11) + np.isnan(data21)
#return time and data
return time1[~bidx], data11[~bidx], data21[~bidx]
def interp_clean_common(time1,data1,time2,data2,filter_max=1000.0,filter_min=-1000.0):
#copy and filter dataset 1
data11=copy.copy(data1)
data11[data1>=filter_max]=np.nan
data11[data1<=filter_min]=np.nan
#copy and filter dataset 2
data22=copy.copy(data2)
data22[data2>=filter_max]=np.nan
data22[data2<=filter_min]=np.nan
# interp filtered data2 to dataset1 times
#print(time2.shape,data22.shape,time1.shape)
data21=ipt.interp1d(time2,data22,time1)
#idx to remove all nans from either dataset
bidx=np.isnan(data11)+np.isnan(data21)
#return time and data
return time1[~bidx],data11[~bidx],data21[~bidx]
idx=np.argwhere((time>=st) & (time<=et))
time=time[idx]
temp=temp[idx]
time=time[~np.isnan(temp)]
temp=temp[~np.isnan(temp)]
inpath='{}/{}_{}/buoy/{}/'.format(datapath,grid,datatype,name)
out=np.load('{}{}_buoy_temp.npy'.format(inpath,name))
out=out[()]
mod=ipt.interp1d(out['time'],out['temp'],time)
obs=temp
stats={}
stats['fvcom_1m']=OrderedDict()
stats['fvcom_1m']=residual_stats(mod,obs)
timeh=dates.drange(dates.num2date(time[0]),dates.num2date(time[-1]),dates.datetime.timedelta(0,3600))
modh=ipt.interp1d(out['time'],out['temp'],timeh)
obsh=ipt.interp1d(time,temp,timeh)
stats['fvcom_60m']=OrderedDict()
stats['fvcom_60m']=residual_stats(modh,obsh)
st = dates.datestr2num('20150501')
et = dates.datestr2num('20151228')
idx = np.argwhere((time >= st) & (time <= et))
time = time[idx]
temp = temp[idx]
time = time[~np.isnan(temp)]
temp = temp[~np.isnan(temp)]
inpath = '{}/{}_{}/buoy/{}/'.format(datapath, grid, datatype, name)
out = np.load('{}{}_buoy_temp.npy'.format(inpath, name))
out = out[()]
mod = ipt.interp1d(out['time'], out['temp'], time)
obs = temp
stats = {}
stats['fvcom_1m'] = OrderedDict()
stats['fvcom_1m'] = residual_stats(mod, obs)
timeh = dates.drange(dates.num2date(time[0]), dates.num2date(time[-1]),
dates.datetime.timedelta(0, 3600))
modh = ipt.interp1d(out['time'], out['temp'], timeh)
obsh = ipt.interp1d(time, temp, timeh)
stats['fvcom_60m'] = OrderedDict()
stats['fvcom_60m'] = residual_stats(modh, obsh)
stats['nemo'] = OrderedDict()
sva = np.empty((len(locs), ))
for i, time in enumerate(times):
lidx = np.argwhere(mtimes <= time).max()
lua = ipt.interpEfield_locs(data, 'ua', locs[i, :], lidx, ll=True)
lva = ipt.interpEfield_locs(data, 'va', locs[i, :], lidx, ll=True)
uidx = np.argwhere(mtimes > time).min()
uua = ipt.interpEfield_locs(data, 'ua', locs[i, :], uidx, ll=True)
uva = ipt.interpEfield_locs(data, 'va', locs[i, :], uidx, ll=True)
#u1 = interp1d(mtimes[[lidx,uidx]], np.array([lua,uua]).flatten())
#sua[i] = u1(time)
#v1 = interp1d(mtimes[[lidx,uidx]], np.array([lva,uva]).flatten())
#sva[i] = v1(time)
sua[i] = ipt.interp1d(mtimes[[lidx, uidx]],
np.array([lua, uua]).flatten(), time)
sva[i] = ipt.interp1d(mtimes[[lidx, uidx]],
np.array([lva, uva]).flatten(), time)
# Create constant spaced versions of sua and sva to plot
ctime = np.linspace(times.min(), times.max(), 100)
csua = ipt.interp1d(times, sua, ctime)
csva = ipt.interp1d(times, sva, ctime)
u_vec = np.empty((len(vidx), ))
v_vec = np.empty((len(vidx), ))
dxs = dx / (np.diff(times) * 3600 * 24)
dys = dy / (np.diff(times) * 3600 * 24)
ctime2 = np.linspace(times.min(), times[:-1].max(), 100)
sva=np.empty((len(locs),))
for i,time in enumerate(times):
lidx=np.argwhere(mtimes<=time).max()
lua=ipt.interpEfield_locs(data,'ua',locs[i,:],lidx,ll=True)
lva=ipt.interpEfield_locs(data,'va',locs[i,:],lidx,ll=True)
uidx=np.argwhere(mtimes>time).min()
uua=ipt.interpEfield_locs(data,'ua',locs[i,:],uidx,ll=True)
uva=ipt.interpEfield_locs(data,'va',locs[i,:],uidx,ll=True)
#u1 = interp1d(mtimes[[lidx,uidx]], np.array([lua,uua]).flatten())
#sua[i] = u1(time)
#v1 = interp1d(mtimes[[lidx,uidx]], np.array([lva,uva]).flatten())
#sva[i] = v1(time)
sua[i] = ipt.interp1d(mtimes[[lidx,uidx]], np.array([lua,uua]).flatten(), time)
sva[i] = ipt.interp1d(mtimes[[lidx,uidx]], np.array([lva,uva]).flatten(), time)
# Create constant spaced versions of sua and sva to plot
ctime=np.linspace(times.min(),times.max(),100)
csua=ipt.interp1d(times,sua,ctime)
csva=ipt.interp1d(times,sva,ctime)
u_vec=np.empty((len(vidx),))
v_vec=np.empty((len(vidx),))
dxs=dx/(np.diff(times)*3600*24)
dys=dy/(np.diff(times)*3600*24)
ctime2=np.linspace(times.min(),times[:-1].max(),100)
cdxs=ipt.interp1d(times[:-1],dxs,ctime2,kind='zero')
ctdm=loadnc('',filename,False)
ctdo = loadnc('{}east/all/'.format(obspath),'ctd_{}.nc'.format(ctdm['ctdnumber'][0]),False)
num=ctdm['ctdnumber'][0]
#try:
#mod=load_ctd_mod('{}ctd_timeseries_{}.txt'.format(modpath,num))
#zeta=load_ctd_zeta_mod('{}ctd_zeta_{}.txt'.format(modpath,num))
#obs=load_ctd_obs('{}{}_ctd.dat'.format(obsp,num))
other['dt']=ctdm['time'][1]-ctdm['time'][0]*24*60
other['num']=num
tidx=np.argmin(np.fabs(ctdm['time']-ctdo['time']))
other['tidx']=tidx
#for i in range(mod['arrays']['depth'].shape[1]):
Tmod=ipt.interp1d(-1*ctdm['siglay']*(ctdm['h']+ctdm['zeta'][tidx]),ctdm['temp'][tidx,:],ctdo['depth'])
Smod=ipt.interp1d(-1*ctdm['siglay']*(ctdm['h']+ctdm['zeta'][tidx]),ctdm['salinity'][tidx,:],ctdo['depth'])
cTmod, cTobs = remove_common_nan(Tmod,ctdo['temp'])
cSmod, cSobs = remove_common_nan(Smod,ctdo['salinity'])
Tstats['{}'.format(num)]=residual_stats(cTmod, cTobs)
Sstats['{}'.format(num)]=residual_stats(cSmod, cSobs)
saveLL['{}'.format(num)]=np.array([ctdo['lon'],ctdo['lat']])
ll=np.empty((len(saveLL.keys()),2))
T=np.empty((len(saveLL.keys()),7))
S=np.empty((len(saveLL.keys()),7))
for i,key in enumerate(saveLL.keys()):
'/home/moflaher/Desktop/workspace_python/dataout/sjh_lr_v1/tg/' + name4,
tgname + '_fvcom.nc', False)
data5 = loadnc(
'/home/moflaher/Desktop/workspace_python/dataout/sjh_lr_v1/tg/' + name5,
tgname + '_fvcom.nc', False)
obs = loadnc('/mnt/drive_1/obs/east/all', tgname + '.nc', False)
tt1 = ttide.t_tide(data1['zeta'], lat=data1['lat'], dt=1)
tt2 = ttide.t_tide(data2['zeta'], lat=data2['lat'], dt=1)
tt4 = ttide.t_tide(data4['zeta'], lat=data4['lat'], dt=1)
tt5 = ttide.t_tide(data5['zeta'], lat=data5['lat'], dt=1)
t1, d1, d2 = interp_clean_common(data2['time'], data2['zeta'], obs['time'],
obs['zeta'], 500, -500)
time = np.arange(t1[0], t1[-1] + 1 / 24.0, 1 / 24.0)
testz = ipt.interp1d(t1, d2, time)
tt3 = ttide.t_tide(testz, lat=obs['lat'], dt=1)
FFT1 = sp.fft(tt1['xres'])
FFT2 = sp.fft(tt2['xres'])
FFT3 = sp.fft(tt3['xres'])
FFT4 = sp.fft(tt4['xres'])
FFT5 = sp.fft(tt5['xres'])
freqs1 = fftp.fftfreq(tt1['xres'].size, 3600)
freqs2 = fftp.fftfreq(tt2['xres'].size, 3600)
freqs3 = fftp.fftfreq(tt3['xres'].size, 3600)
freqs4 = fftp.fftfreq(tt4['xres'].size, 3600)
freqs5 = fftp.fftfreq(tt5['xres'].size, 3600)
f = plt.figure()
other['name']=name
Tstats={}
Sstats={}
for num in ctdnum:
print(num)
try:
mod=load_ctd_mod('{}ctd_timeseries_{}.txt'.format(modpath,num))
obs=load_ctd_obs('{}{}_ctd.dat'.format(obspath,num))
other['num']=num
other['tidx']=np.argmin(np.fabs(mod['arrays']['time'][0,:]-obs['time']))
#for i in range(mod['arrays']['depth'].shape[1]):
Tmod=ipt.interp1d(-1*mod['arrays']['depth'][:,other['tidx']],mod['arrays']['temperature'][:,other['tidx']],obs['Depth'])
Smod=ipt.interp1d(-1*mod['arrays']['depth'][:,other['tidx']],mod['arrays']['salinity'][:,other['tidx']],obs['Depth'])
cTmod, cTobs = remove_common_nan(Tmod,obs['Temp'])
cSmod, cSobs = remove_common_nan(Smod,obs['Salinity'])
Tstats['{}'.format(num)]=residual_stats(cTmod, cTobs)
Sstats['{}'.format(num)]=residual_stats(cSmod, cSobs)
other['filename']='{}ctd_timeseries_{}.png'.format(savepath,num)
plot_tsmap(mod,obs,other)
din = dates.datestr2num(args.dates)
idx = np.argwhere((time >= din[0]) & (time <= din[1]))
time = np.ravel(time[idx])
temp = np.ravel(temp[idx])
figstr = '{}{}_buoy_compare_{}_{}_to_{}.png'.format(
savepath, name, tag, args.dates[0], args.dates[1])
inpath = '{}/{}/buoy/{}/'.format(datapath, grid, name)
out = np.load('{}{}_buoy_temp.npy'.format(inpath, name))
out = out[()]
idx = np.argwhere((out['time'] >= time[0]) & (out['time'] <= time[-1]))
timed = out['time'][idx]
tempd = out['temp'][idx]
itemp = ipt.interp1d(time[~np.isnan(temp)], temp[~np.isnan(temp)], timed)
test = residual_stats(tempd, itemp)
f = plt.figure(figsize=(15, 5))
ax = f.add_axes([.125, .1, .775, .8])
ax.plot(timed, itemp, 'k', label='Buoy')
ax.plot(timed, tempd, lw=.5, label=name)
ax.xaxis.set_major_locator(months)
ax.xaxis.set_major_formatter(monthsFmt)
ax.legend()
#f.suptitle(pd.DataFrame(test).round(2).T.to_string()[15:].replace(' ','').replace('\n',' '))
a = pd.DataFrame(test, index=[0]).round(2).T[0]
f.suptitle(
'Bias: {} Std: {} RMSE: {} RAE: {} Corr: {} Skew: {} Skill: {}'
obs={}
mod={}
obs['rtime']=dates.datestr2num(adcp['time']['Times'])
obs['rzeta']=adcp['pres']['surf']
mod['rtime']=model['time']
mod['rzeta']=model['zeta']
timeshift=dates.drange(dates.num2date(obs['rtime'][0]),dates.num2date(obs['rtime'][-1]),dt)
try:
oz=ipt.interp1d(obs['rtime'],obs['rzeta'],timeshift)
mz=ipt.interp1d(mod['rtime'],mod['rzeta'],timeshift)
nidx=np.isnan(oz)
mz[nidx]=np.nan
oz=oz-np.nanmean(oz)
mz=mz-np.nanmean(mz)
r1z=residual_stats(mz[~nidx],oz[~nidx])
print('T_tide obs')
oout=ttide.t_tide(oz,dt=60.0/3600.0,stime=timeshift[0],lat=adcp['lat'],out_style=None)
window = intv * 24 * 60
savepath = '{}/{}_{}/TG_ttide_window/{}_{}m/'.format(datapath, grid, datatype,
name, window)
if not os.path.exists(savepath): os.makedirs(savepath)
tg65 = np.load('/home/mif001/scratch/obs/tg/tg_yar_clean.npy')
tg65[tg65[:, 1] > 100, 1] = np.nan
tgtime = tg65[:, 0]
tgzeta = tg65[:, 1]
tgzeta1 = tg65[:, 1]
tgtime = tgtime[~np.isnan(tgzeta)]
tgzeta = tgzeta[~np.isnan(tgzeta)]
time = np.arange(tgtime.min(), tgtime.max(), 60 / (24.0 * 3600))
mtgzeta = ipt.interp1d(tgtime.flatten(), tgzeta.flatten(), time)
num = len(
range(0,
len(time) - window,
np.round(intv / 3).astype(int) * 24 * 60))
m2amp = np.empty((num, ))
m2phs = np.empty((num, ))
m2nans = np.empty((num, ))
for j, i in enumerate(
range(0,
len(time) - window,
np.round(intv / 3).astype(int) * 24 * 60)):
out = ttide.t_tide(mtgzeta[starttime + i:window + i],
data1=loadnc('/home/moflaher/Desktop/workspace_python/dataout/sjh_lr_v1_sub/tg/'+name1,tgname+'_fvcom.nc',False)
data2=loadnc('/home/moflaher/Desktop/workspace_python/dataout/sjh_lr_v1_sub/tg/'+name2,tgname+'_fvcom.nc',False)
data4=loadnc('/home/moflaher/Desktop/workspace_python/dataout/sjh_lr_v1/tg/'+name4,tgname+'_fvcom.nc',False)
data5=loadnc('/home/moflaher/Desktop/workspace_python/dataout/sjh_lr_v1/tg/'+name5,tgname+'_fvcom.nc',False)
obs=loadnc('/mnt/drive_1/obs/east/all',tgname+'.nc',False)
tt1=ttide.t_tide(data1['zeta'],lat=data1['lat'],dt=1)
tt2=ttide.t_tide(data2['zeta'],lat=data2['lat'],dt=1)
tt4=ttide.t_tide(data4['zeta'],lat=data4['lat'],dt=1)
tt5=ttide.t_tide(data5['zeta'],lat=data5['lat'],dt=1)
t1,d1,d2=interp_clean_common(data2['time'],data2['zeta'],obs['time'],obs['zeta'],500,-500)
time=np.arange(t1[0],t1[-1]+1/24.0,1/24.0)
testz=ipt.interp1d(t1,d2,time)
tt3=ttide.t_tide(testz,lat=obs['lat'],dt=1)
FFT1=sp.fft(tt1['xres'])
FFT2=sp.fft(tt2['xres'])
FFT3=sp.fft(tt3['xres'])
FFT4=sp.fft(tt4['xres'])
FFT5=sp.fft(tt5['xres'])
freqs1=fftp.fftfreq(tt1['xres'].size,3600)
freqs2=fftp.fftfreq(tt2['xres'].size,3600)
freqs3=fftp.fftfreq(tt3['xres'].size,3600)
freqs4=fftp.fftfreq(tt4['xres'].size,3600)
freqs5=fftp.fftfreq(tt5['xres'].size,3600)
savepath='{}/{}_{}/TG_ttide_window/{}_{}m/'.format(datapath,grid,datatype,name,window)
if not os.path.exists(savepath): os.makedirs(savepath)
tg65=np.load('/home/mif001/scratch/obs/tg/tg_yar_clean.npy')
tg65[tg65[:,1]>100,1]=np.nan
tgtime=tg65[:,0]
tgzeta=tg65[:,1]
tgzeta1=tg65[:,1]
tgtime=tgtime[~np.isnan(tgzeta)]
tgzeta=tgzeta[~np.isnan(tgzeta)]
time=np.arange(tgtime.min(),tgtime.max(),60/(24.0*3600))
mtgzeta=ipt.interp1d(tgtime.flatten(),tgzeta.flatten(),time)
num=len(range(0,len(time)-window,np.round(intv/3).astype(int)*24*60))
m2amp=np.empty((num,))
m2phs=np.empty((num,))
m2nans=np.empty((num,))
for j,i in enumerate(range(0,len(time)-window,np.round(intv/3).astype(int)*24*60)):
out=ttide.t_tide(mtgzeta[starttime+i:window+i],stime=time[starttime+i],dt=np.diff(time)[0]*24,synth=-1,out_style=None)
idx=np.argwhere(out['nameu']=='M2 ')
m2amp[j]=out['tidecon'][idx,0]
m2phs[j]=out['tidecon'][idx,2]
m2nans[j]=np.isnan(tgzeta1[starttime+i:window+i]).sum()
a=pd.DataFrame(stats,index=[0]).round(2).T[0]
f=plt.figure(figsize=(15,5));
ax=f.add_axes([.125,.1,.775,.8]);
ax.plot(time1,data1-np.mean(data1),'k',label='TG: {:05d}'.format(tgm['tgnumber'][0]))
ax.plot(time1,data2-np.mean(data2),'r',lw=.5,label='{}'.format(name))
ax.xaxis.set_major_locator(months)
ax.xaxis.set_major_formatter(monthsFmt)
ax.legend()
ax.set_ylabel('Elevation (m)')
f.suptitle('Removed TG means - Obs: {} Model: {}\n Bias: {} Std: {} RMSE: {} RAE: {} Corr: {} Skew: {} Skill: {}'.format(np.mean(data1),np.mean(data2),a[0],a[1],a[2],a[3],a[4],a[5],a[6]))
f.savefig(figstr,dpi=600)
if tide:
time=np.arange(time1[0],time1[-1]+1/24.0,1/24.0)
tgm_int=ipt.interp1d(tgm['time'],tgm['zeta'],time)
tgonan=tgo['zeta'][idx]
tgonan[tgonan>500]=np.nan
tgo_int=ipt.interp1d(tgo['time'][idx],tgonan,time)
tgm_tcon_pre=ttide.t_tide(tgm_int,stime=time[0],lat=tgm['lat'],dt=(time[1]-time[0])*24.0,out_style=None)
tgo_tcon_pre=ttide.t_tide(tgo_int,stime=time[0],lat=tgm['lat'],dt=(time[1]-time[0])*24.0,out_style=None)
tgm_tcon=ttide.t_tide(tgm_int,stime=time[0],lat=tgm['lat'],dt=(time[1]-time[0])*24.0,constitnames=tgm_tcon_pre['nameu'][tgm_tcon_pre['snr']>=args.snr],out_style=None)
tgo_tcon=ttide.t_tide(tgo_int,stime=time[0],lat=tgm['lat'],dt=(time[1]-time[0])*24.0,constitnames=tgo_tcon_pre['nameu'][tgo_tcon_pre['snr']>=args.snr],out_style=None)
f=plt.figure(figsize=(15,5));
ax=f.add_axes([.125,.1,.775,.8]);
ax.plot(time[:len(tgo_tcon['xres'])],tgo_tcon['xres']-np.nanmean(tgo_tcon['xres']),'k',label='TG: {:05d}'.format(tgm['tgnumber'][0]))
ax.plot(time[:len(tgm_tcon['xres'])],tgm_tcon['xres']-np.nanmean(tgm_tcon['xres']),'r',lw=.5,label='{}'.format(name))
ax.xaxis.set_major_locator(months)
uz=np.empty((len(locs),))
for layer in range(0,mlay):
lu[layer,:]=ipt.interpEfield_locs(data,'u',locs,lidx,ll=True,layer=layer)
lv[layer,:]=ipt.interpEfield_locs(data,'v',locs,lidx,ll=True,layer=layer)
lh=ipt.interpNfield_locs(data,'h',locs,lidx,ll=True)
lz=ipt.interpNfield_locs(data,'zeta',locs,lidx,ll=True)
for i in range(len(mt_idx)-1):
for layer in range(0,mlay):
uu[layer,:]=ipt.interpEfield_locs(data,'u',locs,mt_idx[i+1],ll=True,layer=layer)
uv[layer,:]=ipt.interpEfield_locs(data,'v',locs,mt_idx[i+1],ll=True,layer=layer)
uh=ipt.interpNfield_locs(data,'h',locs,mt_idx[i+1],ll=True)
uz=ipt.interpNfield_locs(data,'zeta',locs,mt_idx[i+1],ll=True)
t_idx=np.argwhere( (time>=mtimes[mt_idx[i]]) & (time<mtimes[mt_idx[i+1]]) )
print(i,len(t_idx))
for idx in t_idx:
su = ipt.interp1d(mtimes[[mt_idx[i],mt_idx[i+1]]], np.squeeze(np.array([lu[:,idx],uu[:,idx]])).T, time[idx])
sv = ipt.interp1d(mtimes[[mt_idx[i],mt_idx[i+1]]], np.squeeze(np.array([lv[:,idx],uv[:,idx]])).T, time[idx])
sh = ipt.interp1d(mtimes[[mt_idx[i],mt_idx[i+1]]], np.squeeze(np.array([lh[idx],uh[idx]])), time[idx])
sz = ipt.interp1d(mtimes[[mt_idx[i],mt_idx[i+1]]], np.squeeze(np.array([lz[idx],uz[idx]])), time[idx])
mtrans[key][line]['u'][:,cnt]=su.flatten()
mtrans[key][line]['v'][:,cnt]=sv.flatten()
mtrans[key][line]['h'][cnt]=sh.flatten()+sz.flatten()
cnt+=1
lu=uu
lv=uv
lh=uh
lz=uz
np.save('data/misc/vhfr_obs/transects/VH_5x1m_corrected_model_vh_high.npy',mtrans)
ctdm = loadnc('', filename, False)
ctdo = loadnc('{}east/all/'.format(obspath),
'ctd_{}.nc'.format(ctdm['ctdnumber'][0]), False)
num = ctdm['ctdnumber'][0]
#try:
#mod=load_ctd_mod('{}ctd_timeseries_{}.txt'.format(modpath,num))
#zeta=load_ctd_zeta_mod('{}ctd_zeta_{}.txt'.format(modpath,num))
#obs=load_ctd_obs('{}{}_ctd.dat'.format(obsp,num))
other['dt'] = ctdm['time'][1] - ctdm['time'][0] * 24 * 60
other['num'] = num
tidx = np.argmin(np.fabs(ctdm['time'] - ctdo['time']))
other['tidx'] = tidx
#for i in range(mod['arrays']['depth'].shape[1]):
Tmod = ipt.interp1d(-1 * ctdm['siglay'] * (ctdm['h'] + ctdm['zeta'][tidx]),
ctdm['temp'][tidx, :], ctdo['depth'])
Smod = ipt.interp1d(-1 * ctdm['siglay'] * (ctdm['h'] + ctdm['zeta'][tidx]),
ctdm['salinity'][tidx, :], ctdo['depth'])
cTmod, cTobs = remove_common_nan(Tmod, ctdo['temp'])
cSmod, cSobs = remove_common_nan(Smod, ctdo['salinity'])
Tstats['{}'.format(num)] = residual_stats(cTmod, cTobs)
Sstats['{}'.format(num)] = residual_stats(cSmod, cSobs)
saveLL['{}'.format(num)] = np.array([ctdo['lon'], ctdo['lat']])
ll = np.empty((len(saveLL.keys()), 2))
T = np.empty((len(saveLL.keys()), 7))
S = np.empty((len(saveLL.keys()), 7))
for i, key in enumerate(saveLL.keys()):
ll[i, ] = saveLL[key][:, 0]
lh = ipt.interpNfield_locs(data, 'h', locs, lidx, ll=True)
lz = ipt.interpNfield_locs(data, 'zeta', locs, lidx, ll=True)
for i in range(len(mt_idx) - 1):
uh = ipt.interpNfield_locs(data, 'h', locs, mt_idx[i + 1], ll=True)
uz = ipt.interpNfield_locs(data,
'zeta',
locs,
mt_idx[i + 1],
ll=True)
t_idx = np.argwhere((timeh >= mtimes[mt_idx[i]])
& (timeh < mtimes[mt_idx[i + 1]]))
print(i, len(t_idx))
for idx in t_idx:
sh = ipt.interp1d(mtimes[[mt_idx[i], mt_idx[i + 1]]],
np.squeeze(np.array([lh[idx], uh[idx]])),
timeh[idx])
sz = ipt.interp1d(mtimes[[mt_idx[i], mt_idx[i + 1]]],
np.squeeze(np.array([lz[idx], uz[idx]])),
timeh[idx])
mtrans[key][line]['h'][cnt] = sh.flatten() + sz.flatten()
cnt += 1
lh = uh
lz = uz
mtrans[key][line]['hc'] = (mtrans[key][line]['h'][:-1] +
mtrans[key][line]['h'][1:]) / 2.0
hsig = -1 * sigh[:, None] * mtrans[key][line]['hc'][None, :]
#get model times for all transect
ctdm=loadnc('',filename,False)
ctdo = loadnc('{}east/all/'.format(obspath),'ctd_{}.nc'.format(ctdm['ctdnumber'][0]),False)
num=ctdm['ctdnumber'][0]
#try:
#mod=load_ctd_mod('{}ctd_timeseries_{}.txt'.format(modpath,num))
#zeta=load_ctd_zeta_mod('{}ctd_zeta_{}.txt'.format(modpath,num))
#obs=load_ctd_obs('{}{}_ctd.dat'.format(obsp,num))
other['dt']=ctdm['time'][1]-ctdm['time'][0]*24*60
other['num']=num
tidx=np.argmin(np.fabs(ctdm['time']-ctdo['time']))
other['tidx']=tidx
#for i in range(mod['arrays']['depth'].shape[1]):
Tmod=ipt.interp1d(-1*ctdm['siglay']*(ctdm['h']+ctdm['zeta'][tidx]),ctdm['temp'][tidx,:],ctdo['depth'])
Smod=ipt.interp1d(-1*ctdm['siglay']*(ctdm['h']+ctdm['zeta'][tidx]),ctdm['salinity'][tidx,:],ctdo['depth'])
cTmod, cTobs = remove_common_nan(Tmod,ctdo['temp'])
cSmod, cSobs = remove_common_nan(Smod,ctdo['salinity'])
Tstats['{}'.format(num)]=residual_stats(cTmod, cTobs)
Sstats['{}'.format(num)]=residual_stats(cSmod, cSobs)
other['filename']='{}ctd_timeseries_{}.png'.format(savepath,num)
#plot_tsmap2(mod,obs,other,Tstats,Sstats)
#compute stats for all times
Tstatst=OrderedDict()
Tmodt=np.empty((len(ctdo['depth'].T),len(ctdm['time'])))
Sstatst=OrderedDict()
Tstats = {}
Sstats = {}
for num in ctdnum:
print(num)
try:
mod = load_ctd_mod('{}ctd_timeseries_{}.txt'.format(modpath, num))
obs = load_ctd_obs('{}{}_ctd.dat'.format(obspath, num))
other['num'] = num
other['tidx'] = np.argmin(
np.fabs(mod['arrays']['time'][0, :] - obs['time']))
#for i in range(mod['arrays']['depth'].shape[1]):
Tmod = ipt.interp1d(-1 * mod['arrays']['depth'][:, other['tidx']],
mod['arrays']['temperature'][:, other['tidx']],
obs['Depth'])
Smod = ipt.interp1d(-1 * mod['arrays']['depth'][:, other['tidx']],
mod['arrays']['salinity'][:, other['tidx']],
obs['Depth'])
cTmod, cTobs = remove_common_nan(Tmod, obs['Temp'])
cSmod, cSobs = remove_common_nan(Smod, obs['Salinity'])
Tstats['{}'.format(num)] = residual_stats(cTmod, cTobs)
Sstats['{}'.format(num)] = residual_stats(cSmod, cSobs)
other['filename'] = '{}ctd_timeseries_{}.png'.format(savepath, num)
plot_tsmap(mod, obs, other)
except:
idx=np.argwhere((time>=din[0]) & (time<=din[1]))
time=np.ravel(time[idx])
temp=np.ravel(temp[idx])
figstr='{}{}_buoy_compare_{}_{}_to_{}.png'.format(savepath,name,tag,args.dates[0],args.dates[1])
inpath='{}/{}/buoy/{}/'.format(datapath,grid,name)
out=np.load('{}{}_buoy_temp.npy'.format(inpath,name))
out=out[()]
idx=np.argwhere((out['time']>=time[0])&(out['time']<=time[-1]))
timed=out['time'][idx]
tempd=out['temp'][idx]
itemp=ipt.interp1d(time[~np.isnan(temp)],temp[~np.isnan(temp)],timed)
test=residual_stats(tempd,itemp)
f=plt.figure(figsize=(15,5))
ax=f.add_axes([.125,.1,.775,.8])
ax.plot(timed, itemp,'k',label='Buoy')
ax.plot(timed,tempd,lw=.5,label=name)
ax.xaxis.set_major_locator(months)
ax.xaxis.set_major_formatter(monthsFmt)
ax.legend()
#f.suptitle(pd.DataFrame(test).round(2).T.to_string()[15:].replace(' ','').replace('\n',' '))
a=pd.DataFrame(test,index=[0]).round(2).T[0]
f.suptitle('Bias: {} Std: {} RMSE: {} RAE: {} Corr: {} Skew: {} Skill: {}'.format(a[0],a[1],a[2],a[3],a[4],a[5],a[6]))
#ax.set_ylabel('SST ($^{\circ}C$)')
lidx=mt_idx[0]
lh=np.empty((len(locs),))
lz=np.empty((len(locs),))
uh=np.empty((len(locs),))
uz=np.empty((len(locs),))
lh=ipt.interpNfield_locs(data,'h',locs,lidx,ll=True)
lz=ipt.interpNfield_locs(data,'zeta',locs,lidx,ll=True)
for i in range(len(mt_idx)-1):
uh=ipt.interpNfield_locs(data,'h',locs,mt_idx[i+1],ll=True)
uz=ipt.interpNfield_locs(data,'zeta',locs,mt_idx[i+1],ll=True)
t_idx=np.argwhere( (timeh>=mtimes[mt_idx[i]]) & (timeh<mtimes[mt_idx[i+1]]) )
print(i,len(t_idx))
for idx in t_idx:
sh = ipt.interp1d(mtimes[[mt_idx[i],mt_idx[i+1]]], np.squeeze(np.array([lh[idx],uh[idx]])), timeh[idx])
sz = ipt.interp1d(mtimes[[mt_idx[i],mt_idx[i+1]]], np.squeeze(np.array([lz[idx],uz[idx]])), timeh[idx])
mtrans[key][line]['h'][cnt]=sh.flatten()+sz.flatten()
cnt+=1
lh=uh
lz=uz
mtrans[key][line]['hc']=(mtrans[key][line]['h'][:-1]+mtrans[key][line]['h'][1:])/2.0
hsig=-1*sigh[:,None]*mtrans[key][line]['hc'][None,:]
#get model times for all transect
mt_idx=np.argwhere( (mtimes> time.min()) & (mtimes<time.max()) )
print(len(mt_idx))
print('')
model = np.load('{}ADCP_{}_model_ministation.npy'.format(lpath,adcp['metadata']['ADCP_number']))
model = model[()]
except:
print('Failed to load {}'.format(adcp['metadata']['ADCP_number']))
continue
obs={}
mod={}
#lvl=np.array([2,5,10])
obs['rtime']=dates.datestr2num(adcp['time']['Times'])
obs['rzeta']=adcp['pres']['surf']
obs['rv']=np.empty((len(obs['rtime']),len(lvl)))
obs['ru']=np.empty((len(obs['rtime']),len(lvl)))
for i in range(len(obs['rtime'])):
obs['rv'][i,]=ipt.interp1d(adcp['pres']['topheight'][i,:],adcp['data']['north_vel'][i,:],lvl)
obs['ru'][i,]=ipt.interp1d(adcp['pres']['topheight'][i,:],adcp['data']['east_vel'][i,:],lvl)
mod['rtime']=model['time']
mod['rzeta']=model['zeta']
mod['rv']=np.empty((len(mod['rtime']),len(lvl)))
mod['ru']=np.empty((len(mod['rtime']),len(lvl)))
for i in range(len(mod['rtime'])):
mod['rv'][i,]=ipt.interp1d(-1*model['siglay']*(model['zeta'][i]+model['h']),model['v'][i,],lvl)
mod['ru'][i,]=ipt.interp1d(-1*model['siglay']*(model['zeta'][i]+model['h']),model['u'][i,],lvl)
ctdm=loadnc('',filename,False)
ctdo = loadnc('{}east/all/'.format(obspath),'ctd_{}.nc'.format(ctdm['ctdnumber'][0]),False)
num=ctdm['ctdnumber'][0]
#try:
#mod=load_ctd_mod('{}ctd_timeseries_{}.txt'.format(modpath,num))
#zeta=load_ctd_zeta_mod('{}ctd_zeta_{}.txt'.format(modpath,num))
#obs=load_ctd_obs('{}{}_ctd.dat'.format(obsp,num))
other['dt']=ctdm['time'][1]-ctdm['time'][0]*24*60
other['num']=num
tidx=np.argmin(np.fabs(ctdm['time']-ctdo['time']))
other['tidx']=tidx
#for i in range(mod['arrays']['depth'].shape[1]):
Tmod=ipt.interp1d(-1*ctdm['siglay']*(ctdm['h']+ctdm['zeta'][tidx]),ctdm['temp'][tidx,:],ctdo['depth'])
Smod=ipt.interp1d(-1*ctdm['siglay']*(ctdm['h']+ctdm['zeta'][tidx]),ctdm['salinity'][tidx,:],ctdo['depth'])
cTmod, cTobs = remove_common_nan(Tmod,ctdo['temp'])
cSmod, cSobs = remove_common_nan(Smod,ctdo['salinity'])
Tstats['{}'.format(num)]=residual_stats(cTmod, cTobs)
Sstats['{}'.format(num)]=residual_stats(cSmod, cSobs)
other['filename']='{}ctd_timeseries_{}.png'.format(savepath,num)
#plot_tsmap2(mod,obs,other,Tstats,Sstats)
#compute stats for all times
Tstatst=OrderedDict()
Tmodt=np.empty((len(ctdo['depth'].T),len(ctdm['time'])))
Sstatst=OrderedDict()
