def funcP(iP, axP):
print(iP)
figP.suptitle('siteNo {} corrLSTM {:.2f}; corrWRTDS {:.2f}'.format(
siteNoLst[iP], xMat[iP], yMat[iP]))
siteNo = siteNoLst[iP]
# ts
v0 = dictObs[siteNo][code].values
v1 = dictLSTM[siteNo][code].values
v2 = dictWRTDS[siteNo][code].values
t = dictObs[siteNo].index.values
legLst = ['LSTM', 'WRTDS', 'Obs']
axplot.plotTS(axP[0],
t[ind1], [v1[ind1], v2[ind1], v0[ind1]],
styLst='--*',
cLst='rbk',
legLst=legLst)
axplot.plotTS(axP[1],
t[ind2], [v1[ind2], v2[ind2], v0[ind2]],
styLst='--*',
cLst='rbk',
legLst=legLst)
# cq
q = dictObs[siteNo]['00060'].values
c = dictObs[siteNo][code].values
td = dictObs[siteNo].index.dayofyear
sc = axP[2].scatter(np.log(q), c, c=td, cmap='hsv', vmin=0, vmax=365)
def onclick(event):
xClick = event.xdata
yClick = event.ydata
d = np.sqrt((xClick - lon)**2 + (yClick - lat)**2)
iS = np.argmin(d)
siteNo = siteNoLstTemp[iS]
for ix in range(nTs):
var = varC[iCLst[ix]]
titleStr = '{} [{}] siteno {}'.format(
codePdf['fullName'][var], codePdf['unit'][var], siteNo,)
obs = dfT[dfT['siteNo'] == siteNo][var].values
print([data[iS] for data in dataLst])
ax = axTsLst[ix]
ax.clear()
t = pd.to_datetime(dfT[dfT['siteNo'] == siteNo]['date']).values
bTrain = dfT[dfT['siteNo'] ==
siteNo]['train'].values.astype(bool)
ind = np.where(~np.isnan(obs))[0]
ind1 = np.where(~np.isnan(obs) & bTrain)[0]
ind2 = np.where(~np.isnan(obs) & ~bTrain)[0]
if len(ind2) != 0:
tBar = t[ind1[-1]]+(t[ind2[0]]-t[ind1[-1]])/2
else:
tBar = None
tt = t[ind]
y = list()
for dfP in dfPLst:
y.append(dfP[dfP['siteNo'] == siteNo][var].values[ind])
y.append(obs[ind])
axplot.plotTS(ax, tt, y, tBar=tBar, cLst='rbgk',
legLst=modelLst+['obs'])
ax.set_title(titleStr)
plt.draw()
def funcPoint(iP, axP):
siteNo = siteNoLst[iP]
dfPred, dfObs = basins.loadSeq(outName, siteNo, ep=200)
dfPred = dfPred[dfPred.index >= np.datetime64('1980-01-01')]
dfObs = dfObs[dfObs.index >= np.datetime64('1980-01-01')]
t = dfPred.index.values.astype(np.datetime64)
tBar = np.datetime64('2000-01-01')
axplot.plotTS(axP[0],
t, [dfPred['00060'], dfObs['00060']],
tBar=tBar,
legLst=['pred', 'obs'],
styLst='--',
cLst='br')
axP[0].set_title('streamflow')
for k, var in enumerate(codeLst):
styLst = '-*'
shortName = codePdf.loc[var]['shortName']
title = ' {} {}'.format(shortName, var)
axplot.plotTS(axP[k + 1],
t, [dfPred[var], dfObs[var]],
tBar=tBar,
legLst=['pred', 'obs'],
styLst=styLst,
cLst='br')
axP[k + 1].set_title(title)
def funcP(axP, iP, iM):
rr = xMat[:, iM]**2-yMat[:, iM]**2
cc = cMat[:, iM] if cMat.ndim == 2 else cMat
dfCrd = gageII.readData(
varLst=['LAT_GAGE', 'LNG_GAGE'], siteNoLst=siteNoLst)
lat = dfCrd['LAT_GAGE'].values
lon = dfCrd['LNG_GAGE'].values
# maps
axplot.mapPoint(axP[0], lat, lon, rr, vRange=[-0.3, 0.3], s=16, cb=False)
circle = plt.Circle([lon[iP], lat[iP]], 2, color='black', fill=False)
axP[0].add_patch(circle)
axplot.mapPoint(axP[1], lat, lon, cc, vRange=cR, s=16, cb=False)
circle = plt.Circle([lon[iP], lat[iP]], 2, color='black', fill=False)
axP[1].add_patch(circle)
siteNo = siteNoLst[iP]
# ts
code = codeLst2[iM]
print(code, siteNo)
print(iP, iM)
v0 = dictObs[siteNo][code].values
v1 = dictLSTM[siteNo][code].values
v2 = dictWRTDS[siteNo][code].values
t = dictObs[siteNo].index.values
legLst = ['LSTM', 'WRTDS', 'Obs']
axplot.plotTS(axP[2], t[ind1], [v1[ind1], v2[ind1], v0[ind1]],
styLst='--*', cLst='rbk', legLst=legLst)
axplot.plotTS(axP[3], t[ind2], [v1[ind2], v2[ind2], v0[ind2]],
styLst='--*', cLst='rbk', legLst=legLst)
# cq
q = dictObs[siteNo]['00060'].values
c = dictObs[siteNo][code].values
td = dictObs[siteNo].index.dayofyear
sc = axP[4].scatter(np.log(q), c, c=td, cmap='hsv', vmin=0, vmax=365)
def funcPoint(iP, axP):
siteNo = siteNoLst[iP]
dfP1, dfObs = basins.loadSeq(outName, siteNo)
rmse1, corr1 = waterQuality.calErrSeq(dfP1[code], dfObs[code])
dfP3 = wqLinear.loadSeq(siteNo, code, 'LR', optT='Y8090')
rmse3, corr3 = waterQuality.calErrSeq(dfP3[code], dfObs[code])
t = dfObs.index.values
tBar = np.datetime64('2000-01-01')
styLst = '-*'
# styLst = ['-', '-*']
dfPLst = [dfP1, dfP3]
rmseLst = [rmse1, rmse3]
corrLst = [corr1, corr3]
for k, dfP in enumerate(dfPLst):
axplot.plotTS(axP[k],
t, [dfP[code], dfObs[code]],
tBar=tBar,
legLst=[modLst[k], 'obs'],
styLst=styLst,
cLst='br')
# ind = np.where(~np.isnan(dfObs[code].values))
# axplot.plotTS(axP[k], t[ind], dfObs[code].values[ind], tBar=tBar,
# legLst=[modLst[k], 'obs'], styLst=styLst, cLst='r')
tStr = '{}, rmse [{:.2f} {:.2f}], corr [{:.2f} {:.2f}]'.format(
siteNo, rmseLst[k][0], rmseLst[k][1], corrLst[k][0], corrLst[k][1])
axP[k].set_title(tStr)
def funcPoint(iP, axP):
siteNo = siteNoLst[iP]
dfO = waterQuality.readSiteTS(siteNo, ['00060', code], freq=freq)
t = dfO.index.values
axplot.plotTS(axP[0], t, dfO['00060'].values, styLst='-*', cLst='bgr')
axplot.plotTS(axP[1], t, dfO[code].values, styLst='*', cLst='bgr')
axP[0].set_title(siteNo)
def funcPoint(iP, axP):
siteNo = siteNoLst[iP]
dfPred, dfObs = basins.loadSeq(outName, siteNo)
t = dfPred['date'].values.astype(np.datetime64)
tBar = np.datetime64('2000-01-01')
# linear model
ind1 = infoTrain[infoTrain['siteNo'] == siteNo].index
[x1, y1, yc1], _ = utils.rmNan([xL1[ind1, :], yL1[ind1, :], ycL1[ind1, :]])
modelY = LinearRegression().fit(x1, y1)
modelYC = LinearRegression().fit(x1, yc1)
sd = np.datetime64('1979-01-01')
ed = np.datetime64('2020-01-01')
dfX = waterQuality.readSiteX(siteNo, sd, ed, varX)
x2 = transform.transInAll(dfX.values, mtdX, statLst=statX)
y2 = modelY.predict(x2)
yc2 = modelYC.predict(x2)
yp = wqData.transOut(y2, statY, varY)
ycp = wqData.transOut(yc2, statYC, varYC)
code = codeLst[0]
axplot.plotTS(axP[0],
t, [dfPred['00060'], yp, dfObs['00060']],
tBar=tBar,
legLst=['lstm', 'lr', 'obs'],
styLst='---',
cLst='bgr')
axplot.plotTS(axP[1],
t, [dfPred[code], ycp, dfObs[code]],
tBar=tBar,
legLst=['lstm', 'lr', 'obs'],
styLst='--*',
cLst='bgr')
def funcPoint(iP, axP):
siteNo = siteNoLst[iP]
dfO = waterQuality.readSiteTS(siteNo, [code], freq=wqData.freq)[code]
t = dfO.index
yr = t.year.values
ind1 = (yr <= 2016) & (yr >= 1980)
ind2 = yr > 2016
o1 = dfO[ind1].values
o2 = dfO[ind2].values
t1 = t[ind1]
t2 = t[ind2]
# LSTM
outName = '{}-{}-{}-{}'.format(dataName, 'comb', label, trainSet)
dfP = basins.loadSeq(outName, siteNo)[code]
# WRTDS
fileWrtds = os.path.join(dirWrtds, 'B16', siteNo)
dfW = pd.read_csv(fileWrtds, index_col=None).set_index('date')[code]
dfW.index = pd.to_datetime(dfW.index)
v1 = [dfP[ind1].values, dfW[ind1].values, o1]
v2 = [dfP[ind2].values, dfW[ind2].values, o2]
axplot.plotTS(axP[0], t1, v1, styLst='--*', cLst='bgr')
axplot.plotTS(axP[1], t2, v2, styLst='--*', cLst='bgr')
# print corr
rmseWRTDS1, corrWRTDS1 = utils.stat.calErr(dfW[ind1].values, o1)
rmseLSTM1, corrLSTM1 = utils.stat.calErr(dfP[ind1].values, o1)
axP[0].set_title('site {} WRTDS {:.2f} LSTM {:.2f}'.format(
siteNo, corrWRTDS1, corrLSTM1))
rmseWRTDS2, corrWRTDS2 = utils.stat.calErr(dfW[ind2].values, o2)
rmseLSTM2, corrLSTM2 = utils.stat.calErr(dfP[ind2].values, o2)
axP[1].set_title('site {} WRTDS {:.2f} LSTM {:.2f}'.format(
siteNo, corrWRTDS2, corrLSTM2))
def funcPoint(iP, axP):
siteNo = siteNoLst[iP]
info1 = wqData.subsetInfo(trainSet)
info2 = wqData.subsetInfo(testSet)
ind1 = info1[info1['siteNo'] == siteNo].index
ind2 = info2[info2['siteNo'] == siteNo].index
t1 = info1['date'][ind1].values.astype(np.datetime64)
t2 = info2['date'][ind2].values.astype(np.datetime64)
t = np.concatenate([t1, t2])
dataLst = list()
styLst = list()
cLst = 'rbgcmy'
for p1, p2, c in zip(pLst1, pLst2, cLst[:len(pLst2)]):
x = np.concatenate([p1[ind1], p2[ind2]])
dataLst.append(x)
styLst.append('--'+c)
y = np.concatenate([o1[ind1], o2[ind2]])
dataLst.append(y)
styLst.append('*k')
tBar = t1[-1]+(t2[0]-t1[-1])/2
for k, code in enumerate(codeSel):
ic = wqData.varC.index(code)
shortName = codePdf.loc[code]['shortName']
title = '{} {} {}'.format(siteNo, shortName, code)
xTS = [x[:, ic] for x in dataLst]
axplot.plotTS(axP[k], t, xTS, styLst=styLst, tBar=tBar,
legLst=['opt1', 'opt2', 'obs'])
axP[k].set_title(title)
def funcP(iP, axP):
siteNo = siteNoLst[iP]
df = dbBasin.readSiteTS(siteNo, varLst=codeP, freq='D')
area = dfCrd.loc[siteNo]['DRAIN_SQKM']
for k, code in enumerate(codeP):
axplot.plotTS(axP[k], df.index, df[code].values, cLst='k')
axP[k].set_title('{} of {} {}'.format(code, siteNo, area))
def funcPoint(iP, axP):
axplot.plotTS(axP,
t, [y[iP, :], x2[iP, :], x[iP, :]],
styLst='*--',
cLst='rgb',
tBar=tBar,
legLst=['SMAP', 'NOAH', 'LSTM'])
def plotP(xx, yy, cc, iP, code):
dfCrd = gageII.readData(
varLst=['LAT_GAGE', 'LNG_GAGE'], siteNoLst=siteNoLst)
lat = dfCrd['LAT_GAGE'].values
lon = dfCrd['LNG_GAGE'].values
# maps
axplot.mapPoint(axP[0], lat, lon, xx, vRange=[-0.5, 1], s=16)
circle = plt.Circle([lon[iP], lat[iP]], 2, color='black', fill=False)
axP[0].add_patch(circle)
axplot.mapPoint(axP[1], lat, lon, yy, vRange=[-0.5, 1], s=16)
circle = plt.Circle([lon[iP], lat[iP]], 2, color='black', fill=False)
axP[1].add_patch(circle)
axplot.mapPoint(axP[2], lat, lon, cc, vRange=cR, s=16)
circle = plt.Circle([lon[iP], lat[iP]], 2, color='black', fill=False)
axP[2].add_patch(circle)
siteNo = siteNoLst[iP]
# ts
v0 = dictObs[siteNo][code].values
v1 = dictLSTM[siteNo][code].values
v2 = dictWRTDS[siteNo][code].values
t = dictObs[siteNo].index.values
legLst = ['LSTM', 'WRTDS', 'Obs']
axplot.plotTS(axP[3], t[ind1], [v1[ind1], v2[ind1], v0[ind1]],
styLst='--*', cLst='rbk', legLst=legLst)
axplot.plotTS(axP[4], t[ind2], [v1[ind2], v2[ind2], v0[ind2]],
styLst='--*', cLst='rbk', legLst=legLst)
# cq
q = dictObs[siteNo]['00060'].values
c = dictObs[siteNo][code].values
td = dictObs[siteNo].index.dayofyear
sc = axP[5].scatter(np.log(q), c, c=td, cmap='hsv', vmin=0, vmax=365)
# figP.colorbar(sc, ax=axP[5])
figP.suptitle('code {} {}; siteNo {} \n corrLSTM {:.2f}; corrWRTDS {:.2f}; {} {}'.format(
code, usgs.codePdf.loc[code]['shortName'], siteNo, xx[iP], yy[iP], cVar, cc[iP]))
figP.show()
def funcPoint(iP, axP):
axplot.plotTS(axP,
t, [data[iP, :] for data in dataTS],
styLst='*--',
cLst='kbr',
tBar=tBar,
legLst=['obs', 'LSTM', 'LSTM-DI'])
def funcPoint(iP, axP):
siteNo = siteNoLst[iP]
dfPred1, dfObs1 = basins.loadSeq(outLst[0], siteNo)
dfPred2, dfObs2 = basins.loadSeq(outLst[1], siteNo)
t = dfPred1['date'].values.astype(np.datetime64)
tBar = np.datetime64('2000-01-01')
axplot.plotTS(axP, t, [dfPred1['00060'], dfPred2['00060'], dfObs2['00060']], tBar=tBar,
legLst=['w/ C', 'w/o C', 'obs'], styLst='---', cLst='bgr')
def funcPoint(iP, axP):
siteNo = siteNoLst[iP]
print(iP, siteNo)
dfO = waterQuality.readSiteTS(siteNo, codeLst + ['00060'], freq='D')
t = dfO.index
for k, code in enumerate(codeLst):
ax = axP[k]
axplot.plotTS(ax, t, dfO[code] * dfO['00060'], styLst='*', cLst='k')
def funcPoint(iP, axP):
siteNo = siteNoLst[iP]
dfC = waterQuality.readSiteY(siteNo, [code])
t = dfC.index.values.astype(np.datetime64)
tBar = np.datetime64('2000-01-01')
axplot.plotTS(axP, t, dfC[code], styLst='*', tBar=tBar)
n1 = dfC[dfC[code].index < tBar].count().values
n2 = dfC[dfC[code].index >= tBar].count().values
axP.set_title('{} #samples = {} {}'.format(siteNo, n1, n2))
def funcPoint(iP, axP):
t = dm.t
tBar = np.datetime64('2010-01-01')
sd = np.datetime64('1980-01-01')
legLst = ['Obs', 'LSTM']
axplot.plotTS(axP,
t, [yO[:, iP, 0], yP[:, iP, 0]],
tBar=tBar,
sd=sd,
styLst='--',
cLst='rb',
legLst=legLst)
axP.legend()
def funcPoint(iP, axP):
axplot.plotTS(axP,
t, [x[iP, :], y[iP, :]],
styLst='-*',
cLst='br',
tBar=tBar,
legLst=['pred', 'obs'])
axP.fill_between(t,
x[iP, :] + sigma[iP, :] * 2,
x[iP, :] - sigma[iP, :] * 2,
facecolor='c',
label=r'95% band')
axP.legend(loc='upper right')
def funcPoint(iP, axP):
siteNo = siteNoLst[iP]
cLst = 'cb'
dfO = waterQuality.readSiteTS(siteNo, [code], freq=wqData.freq)[code]
yr = pd.DatetimeIndex(dfO.index).year
dfO1 = dfO[yr % 2 == 1]
dfO2 = dfO[yr % 2 == 0]
dfC = pd.DataFrame(index=dfO2.dropna().index)
dfC['obs'] = dfO2
for k, label in enumerate(labelLst):
outName = '{}-{}-{}-{}'.format(dataName, code, label, trainSet)
dfP = basins.loadSeq(outName, siteNo)[code]
dfC[label] = dfP
axplot.plotTS(axP, dfP.index, dfP.values, styLst='-', cLst=cLst[k])
axplot.plotTS(axP, dfO1.index, dfO1.values, styLst='*', cLst='m')
axplot.plotTS(axP, dfO2.index, dfO2.values, styLst='*', cLst='r')
axP.legend(labelLst + ['obs train', 'obs test'])
titleStr = 'site {}'.format(siteNo)
for k, label in enumerate(labelLst):
axplot.plotTS(axP,
dfC[label].index,
dfC[label].values,
styLst='*',
cLst=cLst[k])
rmse, corr = utils.stat.calErr(dfC[label].values, dfC['obs'].values)
titleStr = titleStr + ' corr{}={:.3f}'.format(k, corr)
axP.set_title(titleStr)
def funcPoint(iP, axes):
kA = 0
siteNo = siteNoLst[iP]
startDate = pd.datetime(1979, 1, 1)
endDate = pd.datetime(2019, 12, 31)
ctR = pd.date_range(startDate, endDate)
dfData = pd.DataFrame({'date': ctR}).set_index('date')
dfC = usgs.readSample(siteNo, codeLst=codeLst, startDate=startDate)
dfQ = usgs.readStreamflow(siteNo, startDate=startDate)
dfQ = dfQ.rename(columns={'00060_00003': '00060'})
dfData = dfData.join(dfQ)
dfData = dfData.join(dfC)
# plot normalized time series
ax = axes[kA]
kA = kA + 1
t = dfData.index.values
dfDataN = (dfData - dfData.mean()) / dfData.std()
varLst = dfData.columns.tolist()
data = [dfDataN[var].values for var in varLst]
legLst = ['streamflow'
] + [usgs.codePdf.loc[code]['shortName'] for code in codeLst]
axplot.plotTS(ax, t, data, styLst='-***', cLst='krgb', legLst=legLst)
# plot C-Q
nc = len(codeLst)
for k in range(nc):
code = codeLst[k]
q = dfData['00060']
c = dfData[code]
[q, c], ind = utils.rmNan([q, c])
ax = axes[kA]
kA = kA + 1
ax.plot(np.log(q), np.log(c), 'r*')
# plot fractual
for k in range(nc):
code = codeLst[k]
dfV = dfData[dfData[code].notna()]
nt = len(dfData)
x = dfV.index.values.astype('datetime64[D]')
y = dfV[code].values
freq = 2 * np.pi / np.linspace(2, nt, nt)
power = signal.lombscargle(x, y, freq)
ax = axes[kA]
kA = kA + 1
ax.plot(np.log(freq / 2 * np.pi), np.log(power), '-*')
fyr = 2 * np.pi / 365
pyr = signal.lombscargle(x, y, [fyr])
ax.plot(np.log(fyr / 2 * np.pi), np.log(pyr), 'r*')
def funcPoint(iP, axP):
siteNo = siteNoLstCode[iP]
outName1 = '{}-{}-{}-{}'.format(dataName, 'comb', 'QTFP_C', trainSet)
dfL1 = basins.loadSeq(outName1, siteNo)
dfO = waterQuality.readSiteTS(siteNo, [code], freq='W')
t = dfO.index
# ts
tBar = np.datetime64('2010-01-01')
sd = np.datetime64('1980-01-01')
legLst = ['LSTM', 'Obs']
axplot.plotTS(axP, t, [dfL1[code], dfO[code]],
tBar=tBar, sd=sd, styLst='-*', cLst='rk', legLst=legLst)
axP.set_title('site {} corr={:.3f}'.format(siteNo, matMap[iP]))
axP.legend()
def funcPoint(iP, axP):
siteNo = siteNoLst[iP]
dfP1, dfObs = basins.loadSeq(outName, siteNo)
rmse1, corr1 = waterQuality.calErrSeq(dfP1[code], dfObs[code])
t = dfObs.index.values
tBar = np.datetime64('2000-01-01')
axplot.plotTS(axP,
t, [dfP1[code], dfObs[code]],
tBar=tBar,
legLst=['LSTM', 'obs'],
styLst='-*',
cLst='br')
tStr = '{}, rmse [{:.2f} {:.2f}], corr [{:.2f} {:.2f}]'.format(
siteNo, rmse1[0], rmse1[1], corr1[0], corr1[1])
axP.set_title(tStr)
def funcPoint(iP, axP):
[axTS, axH1, axH2, axH3, axP1, axP2] = axP
siteNo = siteNoLstCode[iP]
outName1 = '{}-{}-{}-{}'.format(dataName, 'comb', 'QTFP_C', trainSet)
outName2 = '{}-{}-{}-{}'.format(dataName, 'comb', 'QT_C', trainSet)
dfL1 = basins.loadSeq(outName1, siteNo)
dfL2 = basins.loadSeq(outName2, siteNo)
dfW = pd.read_csv(os.path.join(dirWrtds, 'output', siteNo),
index_col=None).set_index('date')
dfO = waterQuality.readSiteTS(siteNo,
codeLst + ['00060'],
freq=wqData.freq)
dfOD = waterQuality.readSiteTS(siteNo, codeLst + ['00060'], freq='D')
t = dfO.index
# ts
tBar = np.datetime64('2010-01-01')
sd = np.datetime64('1980-01-01')
legLst = ['LSTM QTFP', 'LSTM QT', 'WRTDS', 'Obs']
axplot.plotTS(axTS,
t, [dfL1[code], dfL2[code], dfW[code], dfO[code]],
tBar=tBar,
sd=sd,
styLst='---*',
cLst='mrbk',
legLst=legLst)
corrL = corrMat[indS[iP], iCode, 0]
corrW = corrMat[indS[iP], iCode, 1]
axplot.titleInner(axTS,
'siteNo {} {:.2f} {:.2f}'.format(siteNo, corrL, corrW))
axTS.legend()
# hist
axH1.hist(dfOD[code].values, density=True, bins=50)
axplot.titleInner(axH1, 'histogram {}'.format(shortName))
axH2.hist(dfOD['00060'].values, density=True, bins=50)
axplot.titleInner(axH2, 'histogram {}'.format('Q'))
axH3.hist(np.log(dfOD['00060'].values + 1), density=True, bins=50)
axplot.titleInner(axH3, 'histogram {}'.format('log Q'))
# periodgram
freqQ, powerQ, pQ = calPower('00060', dfOD)
freqC, powerC, pC = calPower(code, dfOD)
axP1.plot(1 / freqQ, powerC, '-*b', label='Periodograms')
axP1.plot(1 / freqQ, pQ, '-*r', label='baluev probability')
axplot.titleInner(axP1, 'streamflow')
axP1.legend()
axP2.plot(1 / freqC, powerC, '-*b', label='Periodograms')
axP2.plot(1 / freqC, pC, '-*r', label='baluev probability')
axplot.titleInner(axP2, shortName)
axP2.legend()
def funcP(axP, iP, iM):
siteNo = siteNoCode[iP]
v1 = dictLSTM[siteNo][code].values
v2 = dictWRTDS[siteNo][code].values
o = dictObs[siteNo][code].values
t = dictObs[siteNo].index.values
iS = siteNoLst.index(siteNo)
# ts
legLst = ['LSTM', 'WRTDS', 'Obs']
axplot.plotTS(axP[1], t[indT1], [v1[indT1], v2[indT1], o[indT1]],
styLst='--*', cLst='rbk', legLst=legLst)
axplot.plotTS(axP[0], t[indT2], [v1[indT2], v2[indT2], o[indT2]],
styLst='--*', cLst='rbk', legLst=legLst)
axP[0].set_title('site {} corrLSTM={:.2f} corrWRTDS={:.2f}'.format(
siteNo, corrMat[iS, ic, 1], corrMat[iS, ic, 2]))
axP[0].legend()
def funcPoint(iP, axP):
siteNo = siteNoLstP[iP]
dfPred1, _ = basins.loadSeq(outLst[0], siteNo)
dfPred2, _ = basins.loadSeq(outLst[1], siteNo)
sd = np.datetime64('1980-01-01')
dfQ = waterQuality.readSiteY(siteNo, ['00060'], sd=sd)
dfC = waterQuality.readSiteY(siteNo,
codeSel + [code + '_cd' for code in codeSel],
sd=sd)
dfPred1 = dfPred1[dfPred1.index >= sd]
dfPred2 = dfPred2[dfPred2.index >= sd]
dfPred1 = dfPred1.multiply(dfPred1['00060'], axis='index')
dfPred2 = dfPred2.multiply(dfPred2['00060'], axis='index')
dfC[codeSel] = dfC[codeSel].multiply(dfQ['00060'], axis='index')
t = dfPred1.index.values.astype(np.datetime64)
# axplot.plotTS(axP[0], t, [dfPred1['00060'], dfQ['00060']], tBar=tBar,
# legLst=['pred-opt1', 'obs'], styLst='--', cLst='br')
# axP[0].set_title('{} streamflow'.format(siteNo))
for k, var in enumerate(codeSel):
shortName = codePdf.loc[var]['shortName']
title = '{} {} {}'.format(siteNo, shortName, var)
styLst = ['-', '-', '*', '*', '*', '*']
legLst = [
'model odd', 'model even', 'obs odd', 'obs even', 'flag even',
'flag odd'
]
yr = dfC.index.year
c1 = dfC[var].values.copy()
c2 = dfC[var].values.copy()
f1 = dfC[var].values.copy()
f2 = dfC[var].values.copy()
vf = dfC[var + '_cd'].values
c1[(vf != 'x') & (vf != 'X')] = np.nan
c1[(yr % 2 == 0)] = np.nan
c2[(vf != 'x') & (vf != 'X')] = np.nan
c2[(yr % 2 == 1)] = np.nan
f1[(vf == 'x') | (vf == 'X') | (yr % 2 == 0)] = np.nan
f2[(vf == 'x') | (vf == 'X') | (yr % 2 == 1)] = np.nan
data = [dfPred1[var].values, dfPred2[var].values, c1, c2, f1, f2]
axplot.plotTS(axP[k],
t,
data,
styLst=styLst,
cLst='bgrmkk',
legLst=legLst)
axP[k].set_title(title)
def funcPoint(iP, axP):
siteNo = siteNoLst[iP]
dfP1, dfObs = basins.loadSeq(outName, siteNo)
t = dfPred.index.values.astype(np.datetime64)
tBar = np.datetime64('2000-01-01')
# Silica
rmse, corr = waterQuality.calErrSeq(dfP1[code], dfObs[code])
axplot.plotTS(axP[0],
t, [dfP1[code], dfObs[code]],
tBar=tBar,
legLst=['LSTM', 'obs'],
styLst='-*',
cLst='br')
tStr = '{}, rmse [{:.2f} {:.2f}], corr [{:.2f} {:.2f}]'.format(
siteNo, rmse[0], rmse[1], corr[0], corr[1])
axP[0].set_title('Silica ' + tStr)
# rm outlier
df = dfObs[dfObs['00955'].notna().values]
y = df['00955'].values
yV = y[y < np.percentile(y, 99)]
yV = yV[yV > np.percentile(y, 1)]
ul = np.mean(yV) + np.std(yV) * 5
dfObs[dfObs['00955'] > ul] = np.nan
# fourier
df = dfObs[dfObs['00955'].notna().values]
# nt = len(dfObs)
nt = 365 * 5
x = (df.index.values.astype('datetime64[D]') -
np.datetime64('1979-01-01')).astype(np.float)
y = df['00955'].values
freq = np.fft.fftfreq(nt)[1:]
ls = LombScargle(x, y)
power = ls.power(freq)
df2 = dfP1['00955']
x2 = (df2.index.values.astype('datetime64[D]') -
np.datetime64('1979-01-01')).astype(np.float)
y2 = df2.values
ls2 = LombScargle(x2, y2)
power2 = ls2.power(freq)
axP[1].set_ylabel('normalize spectrum')
indF = np.where(freq > 0)[0]
axP[1].plot(1 / freq[indF], power2[indF], 'b', label='lstm')
axP[1].plot(1 / freq[indF], power[indF], 'r', label='obs')
axP[1].legend()
axP[1].set_ylabel('power')
axP[1].set_xlabel('period (day)')
def funcPoint(iP, axP):
siteNo = siteNoLst[iP]
dfO = waterQuality.readSiteTS(siteNo, [code], freq=wqData.freq)[code]
t = dfO.index
yr = pd.DatetimeIndex(t).year
o1 = dfO[yr <= 2016].values
o2 = dfO[yr > 2016].values
t1 = t[yr <= 2016]
t2 = t[yr > 2016]
pLst1, pLst2 = (list(), list())
for label in labelLst:
outName = '{}-{}-{}-{}'.format(dataName, 'comb', label, trainSet)
dfP = basins.loadSeq(outName, siteNo)[code]
pLst1.append(dfP[yr <= 2016].values)
pLst2.append(dfP[yr > 2016].values)
axplot.plotTS(axP[0], t1, pLst1 + [o1], styLst='--*', cLst='bgr')
axplot.plotTS(axP[1], t2, pLst2 + [o2], styLst='--*', cLst='bgr')
axP[0].set_title(siteNo)
def funcPoint(iP, axP):
siteNo = siteNoLst[iP]
dfO = waterQuality.readSiteTS(siteNo, [code], freq='W')[code]
t = dfO.index
file1 = os.path.join(dirRoot1, 'output', siteNo)
file2 = os.path.join(dirRoot2, 'output', siteNo)
dfP1 = pd.read_csv(file1, index_col='date')[code]
dfP2 = pd.read_csv(file2, index_col='date')[code]
v = [dfP1.values, dfP2.values, dfO.values]
[v1, v2, o], iv = utils.rmNan([dfP1.values, dfP2.values, dfO.values])
tt = t[iv]
styLst = [['-*'] for x in range(3)]
axplot.plotTS(axP, tt.values, [v1, v2, o], cLst='rbk')
# print corr
rmse1, corr1 = utils.stat.calErr(v[0], v[-1])
rmse2, corr2 = utils.stat.calErr(v[1], v[-1])
axP.set_title('site {} WRTDS {:.2f} only T {:.2f}'.format(
siteNo, corr1, corr2))
def funcPoint(iP, axP):
siteNo = siteNoLst[iP]
dfPred, dfObs = basins.loadSeq(outName, siteNo)
t = dfPred.index.values.astype(np.datetime64)
tBar = np.datetime64('2000-01-01')
info1 = wqData.subsetInfo(trainset)
info2 = wqData.subsetInfo(testset)
ind1 = info1[info1['siteNo'] == siteNo].index
ind2 = info2[info2['siteNo'] == siteNo].index
t1 = info1['date'][ind1].values.astype(np.datetime64)
t2 = info2['date'][ind2].values.astype(np.datetime64)
tp = np.concatenate([t1, t2])
yp = np.concatenate([ycP1[ind1], ycP2[ind2]])
for k, var in enumerate(plotVar):
rmse, corr = waterQuality.calErrSeq(dfPred[var], dfObs[var])
tStr = '{}, rmse [{:.2f} {:.2f}], corr [{:.2f} {:.2f}]'.format(
siteNo, rmse[0], rmse[1], corr[0], corr[1])
if var == '00060':
styLst = '--'
title = 'streamflow ' + tStr
axplot.plotTS(axP[k],
t, [dfPred[var], dfObs[var]],
tBar=tBar,
legLst=['LSTM', 'observation'],
styLst=styLst,
cLst='br')
axP[k].set_title(title)
else:
styLst = '-*'
shortName = codePdf.loc[var]['shortName']
title = shortName + ' ' + tStr
axplot.plotTS(axP[k],
t,
dfPred[var],
tBar=tBar,
legLst=['LSTM-sequence'],
styLst='-',
cLst='b')
axplot.plotTS(axP[k],
tp,
yp,
legLst=['LSTM-sample'],
styLst='*',
cLst='g')
axplot.plotTS(axP[k],
t,
dfObs[var],
legLst=['observation'],
styLst='*',
cLst='r')
axP[k].set_title(title)
def funcP(axP, iP, iM):
siteNo = siteNoCode[iP]
tBar = np.datetime64('2010-01-01')
sd = np.datetime64('1980-01-01')
for k, code in enumerate(codeM):
v1 = dictLSTM[siteNo][code].values
v2 = dictWRTDS[siteNo][code].values
q = dictObs[siteNo]['00060'].values
o = dictObs[siteNo][code].values
t = dictObs[siteNo].index.values
iS = siteNoLst.index(siteNo)
ic = codeLst.index(code)
legLst = ['LSTM', 'WRTDS', 'Obs']
axplot.plotTS(axP[k*2], t, [v1, v2, o], tBar=tBar, sd=sd,
styLst='--*', cLst='rbk', legLst=legLst)
axP[k*2].set_title('site {} corrLSTM={:.2f} corrWRTDS={:.2f}'.format(
siteNo, corrMat[iS, ic, 1], corrMat[iS, ic, 2]))
td = dictObs[siteNo].index.dayofyear
sc = axP[k*2+1].scatter(np.log(q), o, c=td,
cmap='hsv', vmin=0, vmax=365)
