def funcP(iP, axP): print(iP) figP.suptitle('siteNo {} corrLSTM {:.2f}; corrWRTDS {:.2f}'.format( siteNoLst[iP], xMat[iP], yMat[iP])) siteNo = siteNoLst[iP] # ts v0 = dictObs[siteNo][code].values v1 = dictLSTM[siteNo][code].values v2 = dictWRTDS[siteNo][code].values t = dictObs[siteNo].index.values legLst = ['LSTM', 'WRTDS', 'Obs'] axplot.plotTS(axP[0], t[ind1], [v1[ind1], v2[ind1], v0[ind1]], styLst='--*', cLst='rbk', legLst=legLst) axplot.plotTS(axP[1], t[ind2], [v1[ind2], v2[ind2], v0[ind2]], styLst='--*', cLst='rbk', legLst=legLst) # cq q = dictObs[siteNo]['00060'].values c = dictObs[siteNo][code].values td = dictObs[siteNo].index.dayofyear sc = axP[2].scatter(np.log(q), c, c=td, cmap='hsv', vmin=0, vmax=365)
def onclick(event): xClick = event.xdata yClick = event.ydata d = np.sqrt((xClick - lon)**2 + (yClick - lat)**2) iS = np.argmin(d) siteNo = siteNoLstTemp[iS] for ix in range(nTs): var = varC[iCLst[ix]] titleStr = '{} [{}] siteno {}'.format( codePdf['fullName'][var], codePdf['unit'][var], siteNo,) obs = dfT[dfT['siteNo'] == siteNo][var].values print([data[iS] for data in dataLst]) ax = axTsLst[ix] ax.clear() t = pd.to_datetime(dfT[dfT['siteNo'] == siteNo]['date']).values bTrain = dfT[dfT['siteNo'] == siteNo]['train'].values.astype(bool) ind = np.where(~np.isnan(obs))[0] ind1 = np.where(~np.isnan(obs) & bTrain)[0] ind2 = np.where(~np.isnan(obs) & ~bTrain)[0] if len(ind2) != 0: tBar = t[ind1[-1]]+(t[ind2[0]]-t[ind1[-1]])/2 else: tBar = None tt = t[ind] y = list() for dfP in dfPLst: y.append(dfP[dfP['siteNo'] == siteNo][var].values[ind]) y.append(obs[ind]) axplot.plotTS(ax, tt, y, tBar=tBar, cLst='rbgk', legLst=modelLst+['obs']) ax.set_title(titleStr) plt.draw()
def funcPoint(iP, axP): siteNo = siteNoLst[iP] dfPred, dfObs = basins.loadSeq(outName, siteNo, ep=200) dfPred = dfPred[dfPred.index >= np.datetime64('1980-01-01')] dfObs = dfObs[dfObs.index >= np.datetime64('1980-01-01')] t = dfPred.index.values.astype(np.datetime64) tBar = np.datetime64('2000-01-01') axplot.plotTS(axP[0], t, [dfPred['00060'], dfObs['00060']], tBar=tBar, legLst=['pred', 'obs'], styLst='--', cLst='br') axP[0].set_title('streamflow') for k, var in enumerate(codeLst): styLst = '-*' shortName = codePdf.loc[var]['shortName'] title = ' {} {}'.format(shortName, var) axplot.plotTS(axP[k + 1], t, [dfPred[var], dfObs[var]], tBar=tBar, legLst=['pred', 'obs'], styLst=styLst, cLst='br') axP[k + 1].set_title(title)
def funcP(axP, iP, iM): rr = xMat[:, iM]**2-yMat[:, iM]**2 cc = cMat[:, iM] if cMat.ndim == 2 else cMat dfCrd = gageII.readData( varLst=['LAT_GAGE', 'LNG_GAGE'], siteNoLst=siteNoLst) lat = dfCrd['LAT_GAGE'].values lon = dfCrd['LNG_GAGE'].values # maps axplot.mapPoint(axP[0], lat, lon, rr, vRange=[-0.3, 0.3], s=16, cb=False) circle = plt.Circle([lon[iP], lat[iP]], 2, color='black', fill=False) axP[0].add_patch(circle) axplot.mapPoint(axP[1], lat, lon, cc, vRange=cR, s=16, cb=False) circle = plt.Circle([lon[iP], lat[iP]], 2, color='black', fill=False) axP[1].add_patch(circle) siteNo = siteNoLst[iP] # ts code = codeLst2[iM] print(code, siteNo) print(iP, iM) v0 = dictObs[siteNo][code].values v1 = dictLSTM[siteNo][code].values v2 = dictWRTDS[siteNo][code].values t = dictObs[siteNo].index.values legLst = ['LSTM', 'WRTDS', 'Obs'] axplot.plotTS(axP[2], t[ind1], [v1[ind1], v2[ind1], v0[ind1]], styLst='--*', cLst='rbk', legLst=legLst) axplot.plotTS(axP[3], t[ind2], [v1[ind2], v2[ind2], v0[ind2]], styLst='--*', cLst='rbk', legLst=legLst) # cq q = dictObs[siteNo]['00060'].values c = dictObs[siteNo][code].values td = dictObs[siteNo].index.dayofyear sc = axP[4].scatter(np.log(q), c, c=td, cmap='hsv', vmin=0, vmax=365)
def funcPoint(iP, axP): siteNo = siteNoLst[iP] dfP1, dfObs = basins.loadSeq(outName, siteNo) rmse1, corr1 = waterQuality.calErrSeq(dfP1[code], dfObs[code]) dfP3 = wqLinear.loadSeq(siteNo, code, 'LR', optT='Y8090') rmse3, corr3 = waterQuality.calErrSeq(dfP3[code], dfObs[code]) t = dfObs.index.values tBar = np.datetime64('2000-01-01') styLst = '-*' # styLst = ['-', '-*'] dfPLst = [dfP1, dfP3] rmseLst = [rmse1, rmse3] corrLst = [corr1, corr3] for k, dfP in enumerate(dfPLst): axplot.plotTS(axP[k], t, [dfP[code], dfObs[code]], tBar=tBar, legLst=[modLst[k], 'obs'], styLst=styLst, cLst='br') # ind = np.where(~np.isnan(dfObs[code].values)) # axplot.plotTS(axP[k], t[ind], dfObs[code].values[ind], tBar=tBar, # legLst=[modLst[k], 'obs'], styLst=styLst, cLst='r') tStr = '{}, rmse [{:.2f} {:.2f}], corr [{:.2f} {:.2f}]'.format( siteNo, rmseLst[k][0], rmseLst[k][1], corrLst[k][0], corrLst[k][1]) axP[k].set_title(tStr)
def funcPoint(iP, axP): siteNo = siteNoLst[iP] dfO = waterQuality.readSiteTS(siteNo, ['00060', code], freq=freq) t = dfO.index.values axplot.plotTS(axP[0], t, dfO['00060'].values, styLst='-*', cLst='bgr') axplot.plotTS(axP[1], t, dfO[code].values, styLst='*', cLst='bgr') axP[0].set_title(siteNo)
def funcPoint(iP, axP): siteNo = siteNoLst[iP] dfPred, dfObs = basins.loadSeq(outName, siteNo) t = dfPred['date'].values.astype(np.datetime64) tBar = np.datetime64('2000-01-01') # linear model ind1 = infoTrain[infoTrain['siteNo'] == siteNo].index [x1, y1, yc1], _ = utils.rmNan([xL1[ind1, :], yL1[ind1, :], ycL1[ind1, :]]) modelY = LinearRegression().fit(x1, y1) modelYC = LinearRegression().fit(x1, yc1) sd = np.datetime64('1979-01-01') ed = np.datetime64('2020-01-01') dfX = waterQuality.readSiteX(siteNo, sd, ed, varX) x2 = transform.transInAll(dfX.values, mtdX, statLst=statX) y2 = modelY.predict(x2) yc2 = modelYC.predict(x2) yp = wqData.transOut(y2, statY, varY) ycp = wqData.transOut(yc2, statYC, varYC) code = codeLst[0] axplot.plotTS(axP[0], t, [dfPred['00060'], yp, dfObs['00060']], tBar=tBar, legLst=['lstm', 'lr', 'obs'], styLst='---', cLst='bgr') axplot.plotTS(axP[1], t, [dfPred[code], ycp, dfObs[code]], tBar=tBar, legLst=['lstm', 'lr', 'obs'], styLst='--*', cLst='bgr')
def funcPoint(iP, axP): siteNo = siteNoLst[iP] dfO = waterQuality.readSiteTS(siteNo, [code], freq=wqData.freq)[code] t = dfO.index yr = t.year.values ind1 = (yr <= 2016) & (yr >= 1980) ind2 = yr > 2016 o1 = dfO[ind1].values o2 = dfO[ind2].values t1 = t[ind1] t2 = t[ind2] # LSTM outName = '{}-{}-{}-{}'.format(dataName, 'comb', label, trainSet) dfP = basins.loadSeq(outName, siteNo)[code] # WRTDS fileWrtds = os.path.join(dirWrtds, 'B16', siteNo) dfW = pd.read_csv(fileWrtds, index_col=None).set_index('date')[code] dfW.index = pd.to_datetime(dfW.index) v1 = [dfP[ind1].values, dfW[ind1].values, o1] v2 = [dfP[ind2].values, dfW[ind2].values, o2] axplot.plotTS(axP[0], t1, v1, styLst='--*', cLst='bgr') axplot.plotTS(axP[1], t2, v2, styLst='--*', cLst='bgr') # print corr rmseWRTDS1, corrWRTDS1 = utils.stat.calErr(dfW[ind1].values, o1) rmseLSTM1, corrLSTM1 = utils.stat.calErr(dfP[ind1].values, o1) axP[0].set_title('site {} WRTDS {:.2f} LSTM {:.2f}'.format( siteNo, corrWRTDS1, corrLSTM1)) rmseWRTDS2, corrWRTDS2 = utils.stat.calErr(dfW[ind2].values, o2) rmseLSTM2, corrLSTM2 = utils.stat.calErr(dfP[ind2].values, o2) axP[1].set_title('site {} WRTDS {:.2f} LSTM {:.2f}'.format( siteNo, corrWRTDS2, corrLSTM2))
def funcPoint(iP, axP): siteNo = siteNoLst[iP] info1 = wqData.subsetInfo(trainSet) info2 = wqData.subsetInfo(testSet) ind1 = info1[info1['siteNo'] == siteNo].index ind2 = info2[info2['siteNo'] == siteNo].index t1 = info1['date'][ind1].values.astype(np.datetime64) t2 = info2['date'][ind2].values.astype(np.datetime64) t = np.concatenate([t1, t2]) dataLst = list() styLst = list() cLst = 'rbgcmy' for p1, p2, c in zip(pLst1, pLst2, cLst[:len(pLst2)]): x = np.concatenate([p1[ind1], p2[ind2]]) dataLst.append(x) styLst.append('--'+c) y = np.concatenate([o1[ind1], o2[ind2]]) dataLst.append(y) styLst.append('*k') tBar = t1[-1]+(t2[0]-t1[-1])/2 for k, code in enumerate(codeSel): ic = wqData.varC.index(code) shortName = codePdf.loc[code]['shortName'] title = '{} {} {}'.format(siteNo, shortName, code) xTS = [x[:, ic] for x in dataLst] axplot.plotTS(axP[k], t, xTS, styLst=styLst, tBar=tBar, legLst=['opt1', 'opt2', 'obs']) axP[k].set_title(title)
def funcP(iP, axP): siteNo = siteNoLst[iP] df = dbBasin.readSiteTS(siteNo, varLst=codeP, freq='D') area = dfCrd.loc[siteNo]['DRAIN_SQKM'] for k, code in enumerate(codeP): axplot.plotTS(axP[k], df.index, df[code].values, cLst='k') axP[k].set_title('{} of {} {}'.format(code, siteNo, area))
def funcPoint(iP, axP): axplot.plotTS(axP, t, [y[iP, :], x2[iP, :], x[iP, :]], styLst='*--', cLst='rgb', tBar=tBar, legLst=['SMAP', 'NOAH', 'LSTM'])
def plotP(xx, yy, cc, iP, code): dfCrd = gageII.readData( varLst=['LAT_GAGE', 'LNG_GAGE'], siteNoLst=siteNoLst) lat = dfCrd['LAT_GAGE'].values lon = dfCrd['LNG_GAGE'].values # maps axplot.mapPoint(axP[0], lat, lon, xx, vRange=[-0.5, 1], s=16) circle = plt.Circle([lon[iP], lat[iP]], 2, color='black', fill=False) axP[0].add_patch(circle) axplot.mapPoint(axP[1], lat, lon, yy, vRange=[-0.5, 1], s=16) circle = plt.Circle([lon[iP], lat[iP]], 2, color='black', fill=False) axP[1].add_patch(circle) axplot.mapPoint(axP[2], lat, lon, cc, vRange=cR, s=16) circle = plt.Circle([lon[iP], lat[iP]], 2, color='black', fill=False) axP[2].add_patch(circle) siteNo = siteNoLst[iP] # ts v0 = dictObs[siteNo][code].values v1 = dictLSTM[siteNo][code].values v2 = dictWRTDS[siteNo][code].values t = dictObs[siteNo].index.values legLst = ['LSTM', 'WRTDS', 'Obs'] axplot.plotTS(axP[3], t[ind1], [v1[ind1], v2[ind1], v0[ind1]], styLst='--*', cLst='rbk', legLst=legLst) axplot.plotTS(axP[4], t[ind2], [v1[ind2], v2[ind2], v0[ind2]], styLst='--*', cLst='rbk', legLst=legLst) # cq q = dictObs[siteNo]['00060'].values c = dictObs[siteNo][code].values td = dictObs[siteNo].index.dayofyear sc = axP[5].scatter(np.log(q), c, c=td, cmap='hsv', vmin=0, vmax=365) # figP.colorbar(sc, ax=axP[5]) figP.suptitle('code {} {}; siteNo {} \n corrLSTM {:.2f}; corrWRTDS {:.2f}; {} {}'.format( code, usgs.codePdf.loc[code]['shortName'], siteNo, xx[iP], yy[iP], cVar, cc[iP])) figP.show()
def funcPoint(iP, axP): axplot.plotTS(axP, t, [data[iP, :] for data in dataTS], styLst='*--', cLst='kbr', tBar=tBar, legLst=['obs', 'LSTM', 'LSTM-DI'])
def funcPoint(iP, axP): siteNo = siteNoLst[iP] dfPred1, dfObs1 = basins.loadSeq(outLst[0], siteNo) dfPred2, dfObs2 = basins.loadSeq(outLst[1], siteNo) t = dfPred1['date'].values.astype(np.datetime64) tBar = np.datetime64('2000-01-01') axplot.plotTS(axP, t, [dfPred1['00060'], dfPred2['00060'], dfObs2['00060']], tBar=tBar, legLst=['w/ C', 'w/o C', 'obs'], styLst='---', cLst='bgr')
def funcPoint(iP, axP): siteNo = siteNoLst[iP] print(iP, siteNo) dfO = waterQuality.readSiteTS(siteNo, codeLst + ['00060'], freq='D') t = dfO.index for k, code in enumerate(codeLst): ax = axP[k] axplot.plotTS(ax, t, dfO[code] * dfO['00060'], styLst='*', cLst='k')
def funcPoint(iP, axP): siteNo = siteNoLst[iP] dfC = waterQuality.readSiteY(siteNo, [code]) t = dfC.index.values.astype(np.datetime64) tBar = np.datetime64('2000-01-01') axplot.plotTS(axP, t, dfC[code], styLst='*', tBar=tBar) n1 = dfC[dfC[code].index < tBar].count().values n2 = dfC[dfC[code].index >= tBar].count().values axP.set_title('{} #samples = {} {}'.format(siteNo, n1, n2))
def funcPoint(iP, axP): t = dm.t tBar = np.datetime64('2010-01-01') sd = np.datetime64('1980-01-01') legLst = ['Obs', 'LSTM'] axplot.plotTS(axP, t, [yO[:, iP, 0], yP[:, iP, 0]], tBar=tBar, sd=sd, styLst='--', cLst='rb', legLst=legLst) axP.legend()
def funcPoint(iP, axP): axplot.plotTS(axP, t, [x[iP, :], y[iP, :]], styLst='-*', cLst='br', tBar=tBar, legLst=['pred', 'obs']) axP.fill_between(t, x[iP, :] + sigma[iP, :] * 2, x[iP, :] - sigma[iP, :] * 2, facecolor='c', label=r'95% band') axP.legend(loc='upper right')
def funcPoint(iP, axP): siteNo = siteNoLst[iP] cLst = 'cb' dfO = waterQuality.readSiteTS(siteNo, [code], freq=wqData.freq)[code] yr = pd.DatetimeIndex(dfO.index).year dfO1 = dfO[yr % 2 == 1] dfO2 = dfO[yr % 2 == 0] dfC = pd.DataFrame(index=dfO2.dropna().index) dfC['obs'] = dfO2 for k, label in enumerate(labelLst): outName = '{}-{}-{}-{}'.format(dataName, code, label, trainSet) dfP = basins.loadSeq(outName, siteNo)[code] dfC[label] = dfP axplot.plotTS(axP, dfP.index, dfP.values, styLst='-', cLst=cLst[k]) axplot.plotTS(axP, dfO1.index, dfO1.values, styLst='*', cLst='m') axplot.plotTS(axP, dfO2.index, dfO2.values, styLst='*', cLst='r') axP.legend(labelLst + ['obs train', 'obs test']) titleStr = 'site {}'.format(siteNo) for k, label in enumerate(labelLst): axplot.plotTS(axP, dfC[label].index, dfC[label].values, styLst='*', cLst=cLst[k]) rmse, corr = utils.stat.calErr(dfC[label].values, dfC['obs'].values) titleStr = titleStr + ' corr{}={:.3f}'.format(k, corr) axP.set_title(titleStr)
def funcPoint(iP, axes): kA = 0 siteNo = siteNoLst[iP] startDate = pd.datetime(1979, 1, 1) endDate = pd.datetime(2019, 12, 31) ctR = pd.date_range(startDate, endDate) dfData = pd.DataFrame({'date': ctR}).set_index('date') dfC = usgs.readSample(siteNo, codeLst=codeLst, startDate=startDate) dfQ = usgs.readStreamflow(siteNo, startDate=startDate) dfQ = dfQ.rename(columns={'00060_00003': '00060'}) dfData = dfData.join(dfQ) dfData = dfData.join(dfC) # plot normalized time series ax = axes[kA] kA = kA + 1 t = dfData.index.values dfDataN = (dfData - dfData.mean()) / dfData.std() varLst = dfData.columns.tolist() data = [dfDataN[var].values for var in varLst] legLst = ['streamflow' ] + [usgs.codePdf.loc[code]['shortName'] for code in codeLst] axplot.plotTS(ax, t, data, styLst='-***', cLst='krgb', legLst=legLst) # plot C-Q nc = len(codeLst) for k in range(nc): code = codeLst[k] q = dfData['00060'] c = dfData[code] [q, c], ind = utils.rmNan([q, c]) ax = axes[kA] kA = kA + 1 ax.plot(np.log(q), np.log(c), 'r*') # plot fractual for k in range(nc): code = codeLst[k] dfV = dfData[dfData[code].notna()] nt = len(dfData) x = dfV.index.values.astype('datetime64[D]') y = dfV[code].values freq = 2 * np.pi / np.linspace(2, nt, nt) power = signal.lombscargle(x, y, freq) ax = axes[kA] kA = kA + 1 ax.plot(np.log(freq / 2 * np.pi), np.log(power), '-*') fyr = 2 * np.pi / 365 pyr = signal.lombscargle(x, y, [fyr]) ax.plot(np.log(fyr / 2 * np.pi), np.log(pyr), 'r*')
def funcPoint(iP, axP): siteNo = siteNoLstCode[iP] outName1 = '{}-{}-{}-{}'.format(dataName, 'comb', 'QTFP_C', trainSet) dfL1 = basins.loadSeq(outName1, siteNo) dfO = waterQuality.readSiteTS(siteNo, [code], freq='W') t = dfO.index # ts tBar = np.datetime64('2010-01-01') sd = np.datetime64('1980-01-01') legLst = ['LSTM', 'Obs'] axplot.plotTS(axP, t, [dfL1[code], dfO[code]], tBar=tBar, sd=sd, styLst='-*', cLst='rk', legLst=legLst) axP.set_title('site {} corr={:.3f}'.format(siteNo, matMap[iP])) axP.legend()
def funcPoint(iP, axP): siteNo = siteNoLst[iP] dfP1, dfObs = basins.loadSeq(outName, siteNo) rmse1, corr1 = waterQuality.calErrSeq(dfP1[code], dfObs[code]) t = dfObs.index.values tBar = np.datetime64('2000-01-01') axplot.plotTS(axP, t, [dfP1[code], dfObs[code]], tBar=tBar, legLst=['LSTM', 'obs'], styLst='-*', cLst='br') tStr = '{}, rmse [{:.2f} {:.2f}], corr [{:.2f} {:.2f}]'.format( siteNo, rmse1[0], rmse1[1], corr1[0], corr1[1]) axP.set_title(tStr)
def funcPoint(iP, axP): [axTS, axH1, axH2, axH3, axP1, axP2] = axP siteNo = siteNoLstCode[iP] outName1 = '{}-{}-{}-{}'.format(dataName, 'comb', 'QTFP_C', trainSet) outName2 = '{}-{}-{}-{}'.format(dataName, 'comb', 'QT_C', trainSet) dfL1 = basins.loadSeq(outName1, siteNo) dfL2 = basins.loadSeq(outName2, siteNo) dfW = pd.read_csv(os.path.join(dirWrtds, 'output', siteNo), index_col=None).set_index('date') dfO = waterQuality.readSiteTS(siteNo, codeLst + ['00060'], freq=wqData.freq) dfOD = waterQuality.readSiteTS(siteNo, codeLst + ['00060'], freq='D') t = dfO.index # ts tBar = np.datetime64('2010-01-01') sd = np.datetime64('1980-01-01') legLst = ['LSTM QTFP', 'LSTM QT', 'WRTDS', 'Obs'] axplot.plotTS(axTS, t, [dfL1[code], dfL2[code], dfW[code], dfO[code]], tBar=tBar, sd=sd, styLst='---*', cLst='mrbk', legLst=legLst) corrL = corrMat[indS[iP], iCode, 0] corrW = corrMat[indS[iP], iCode, 1] axplot.titleInner(axTS, 'siteNo {} {:.2f} {:.2f}'.format(siteNo, corrL, corrW)) axTS.legend() # hist axH1.hist(dfOD[code].values, density=True, bins=50) axplot.titleInner(axH1, 'histogram {}'.format(shortName)) axH2.hist(dfOD['00060'].values, density=True, bins=50) axplot.titleInner(axH2, 'histogram {}'.format('Q')) axH3.hist(np.log(dfOD['00060'].values + 1), density=True, bins=50) axplot.titleInner(axH3, 'histogram {}'.format('log Q')) # periodgram freqQ, powerQ, pQ = calPower('00060', dfOD) freqC, powerC, pC = calPower(code, dfOD) axP1.plot(1 / freqQ, powerC, '-*b', label='Periodograms') axP1.plot(1 / freqQ, pQ, '-*r', label='baluev probability') axplot.titleInner(axP1, 'streamflow') axP1.legend() axP2.plot(1 / freqC, powerC, '-*b', label='Periodograms') axP2.plot(1 / freqC, pC, '-*r', label='baluev probability') axplot.titleInner(axP2, shortName) axP2.legend()
def funcP(axP, iP, iM): siteNo = siteNoCode[iP] v1 = dictLSTM[siteNo][code].values v2 = dictWRTDS[siteNo][code].values o = dictObs[siteNo][code].values t = dictObs[siteNo].index.values iS = siteNoLst.index(siteNo) # ts legLst = ['LSTM', 'WRTDS', 'Obs'] axplot.plotTS(axP[1], t[indT1], [v1[indT1], v2[indT1], o[indT1]], styLst='--*', cLst='rbk', legLst=legLst) axplot.plotTS(axP[0], t[indT2], [v1[indT2], v2[indT2], o[indT2]], styLst='--*', cLst='rbk', legLst=legLst) axP[0].set_title('site {} corrLSTM={:.2f} corrWRTDS={:.2f}'.format( siteNo, corrMat[iS, ic, 1], corrMat[iS, ic, 2])) axP[0].legend()
def funcPoint(iP, axP): siteNo = siteNoLstP[iP] dfPred1, _ = basins.loadSeq(outLst[0], siteNo) dfPred2, _ = basins.loadSeq(outLst[1], siteNo) sd = np.datetime64('1980-01-01') dfQ = waterQuality.readSiteY(siteNo, ['00060'], sd=sd) dfC = waterQuality.readSiteY(siteNo, codeSel + [code + '_cd' for code in codeSel], sd=sd) dfPred1 = dfPred1[dfPred1.index >= sd] dfPred2 = dfPred2[dfPred2.index >= sd] dfPred1 = dfPred1.multiply(dfPred1['00060'], axis='index') dfPred2 = dfPred2.multiply(dfPred2['00060'], axis='index') dfC[codeSel] = dfC[codeSel].multiply(dfQ['00060'], axis='index') t = dfPred1.index.values.astype(np.datetime64) # axplot.plotTS(axP[0], t, [dfPred1['00060'], dfQ['00060']], tBar=tBar, # legLst=['pred-opt1', 'obs'], styLst='--', cLst='br') # axP[0].set_title('{} streamflow'.format(siteNo)) for k, var in enumerate(codeSel): shortName = codePdf.loc[var]['shortName'] title = '{} {} {}'.format(siteNo, shortName, var) styLst = ['-', '-', '*', '*', '*', '*'] legLst = [ 'model odd', 'model even', 'obs odd', 'obs even', 'flag even', 'flag odd' ] yr = dfC.index.year c1 = dfC[var].values.copy() c2 = dfC[var].values.copy() f1 = dfC[var].values.copy() f2 = dfC[var].values.copy() vf = dfC[var + '_cd'].values c1[(vf != 'x') & (vf != 'X')] = np.nan c1[(yr % 2 == 0)] = np.nan c2[(vf != 'x') & (vf != 'X')] = np.nan c2[(yr % 2 == 1)] = np.nan f1[(vf == 'x') | (vf == 'X') | (yr % 2 == 0)] = np.nan f2[(vf == 'x') | (vf == 'X') | (yr % 2 == 1)] = np.nan data = [dfPred1[var].values, dfPred2[var].values, c1, c2, f1, f2] axplot.plotTS(axP[k], t, data, styLst=styLst, cLst='bgrmkk', legLst=legLst) axP[k].set_title(title)
def funcPoint(iP, axP): siteNo = siteNoLst[iP] dfP1, dfObs = basins.loadSeq(outName, siteNo) t = dfPred.index.values.astype(np.datetime64) tBar = np.datetime64('2000-01-01') # Silica rmse, corr = waterQuality.calErrSeq(dfP1[code], dfObs[code]) axplot.plotTS(axP[0], t, [dfP1[code], dfObs[code]], tBar=tBar, legLst=['LSTM', 'obs'], styLst='-*', cLst='br') tStr = '{}, rmse [{:.2f} {:.2f}], corr [{:.2f} {:.2f}]'.format( siteNo, rmse[0], rmse[1], corr[0], corr[1]) axP[0].set_title('Silica ' + tStr) # rm outlier df = dfObs[dfObs['00955'].notna().values] y = df['00955'].values yV = y[y < np.percentile(y, 99)] yV = yV[yV > np.percentile(y, 1)] ul = np.mean(yV) + np.std(yV) * 5 dfObs[dfObs['00955'] > ul] = np.nan # fourier df = dfObs[dfObs['00955'].notna().values] # nt = len(dfObs) nt = 365 * 5 x = (df.index.values.astype('datetime64[D]') - np.datetime64('1979-01-01')).astype(np.float) y = df['00955'].values freq = np.fft.fftfreq(nt)[1:] ls = LombScargle(x, y) power = ls.power(freq) df2 = dfP1['00955'] x2 = (df2.index.values.astype('datetime64[D]') - np.datetime64('1979-01-01')).astype(np.float) y2 = df2.values ls2 = LombScargle(x2, y2) power2 = ls2.power(freq) axP[1].set_ylabel('normalize spectrum') indF = np.where(freq > 0)[0] axP[1].plot(1 / freq[indF], power2[indF], 'b', label='lstm') axP[1].plot(1 / freq[indF], power[indF], 'r', label='obs') axP[1].legend() axP[1].set_ylabel('power') axP[1].set_xlabel('period (day)')
def funcPoint(iP, axP): siteNo = siteNoLst[iP] dfO = waterQuality.readSiteTS(siteNo, [code], freq=wqData.freq)[code] t = dfO.index yr = pd.DatetimeIndex(t).year o1 = dfO[yr <= 2016].values o2 = dfO[yr > 2016].values t1 = t[yr <= 2016] t2 = t[yr > 2016] pLst1, pLst2 = (list(), list()) for label in labelLst: outName = '{}-{}-{}-{}'.format(dataName, 'comb', label, trainSet) dfP = basins.loadSeq(outName, siteNo)[code] pLst1.append(dfP[yr <= 2016].values) pLst2.append(dfP[yr > 2016].values) axplot.plotTS(axP[0], t1, pLst1 + [o1], styLst='--*', cLst='bgr') axplot.plotTS(axP[1], t2, pLst2 + [o2], styLst='--*', cLst='bgr') axP[0].set_title(siteNo)
def funcPoint(iP, axP): siteNo = siteNoLst[iP] dfO = waterQuality.readSiteTS(siteNo, [code], freq='W')[code] t = dfO.index file1 = os.path.join(dirRoot1, 'output', siteNo) file2 = os.path.join(dirRoot2, 'output', siteNo) dfP1 = pd.read_csv(file1, index_col='date')[code] dfP2 = pd.read_csv(file2, index_col='date')[code] v = [dfP1.values, dfP2.values, dfO.values] [v1, v2, o], iv = utils.rmNan([dfP1.values, dfP2.values, dfO.values]) tt = t[iv] styLst = [['-*'] for x in range(3)] axplot.plotTS(axP, tt.values, [v1, v2, o], cLst='rbk') # print corr rmse1, corr1 = utils.stat.calErr(v[0], v[-1]) rmse2, corr2 = utils.stat.calErr(v[1], v[-1]) axP.set_title('site {} WRTDS {:.2f} only T {:.2f}'.format( siteNo, corr1, corr2))
def funcPoint(iP, axP): siteNo = siteNoLst[iP] dfPred, dfObs = basins.loadSeq(outName, siteNo) t = dfPred.index.values.astype(np.datetime64) tBar = np.datetime64('2000-01-01') info1 = wqData.subsetInfo(trainset) info2 = wqData.subsetInfo(testset) ind1 = info1[info1['siteNo'] == siteNo].index ind2 = info2[info2['siteNo'] == siteNo].index t1 = info1['date'][ind1].values.astype(np.datetime64) t2 = info2['date'][ind2].values.astype(np.datetime64) tp = np.concatenate([t1, t2]) yp = np.concatenate([ycP1[ind1], ycP2[ind2]]) for k, var in enumerate(plotVar): rmse, corr = waterQuality.calErrSeq(dfPred[var], dfObs[var]) tStr = '{}, rmse [{:.2f} {:.2f}], corr [{:.2f} {:.2f}]'.format( siteNo, rmse[0], rmse[1], corr[0], corr[1]) if var == '00060': styLst = '--' title = 'streamflow ' + tStr axplot.plotTS(axP[k], t, [dfPred[var], dfObs[var]], tBar=tBar, legLst=['LSTM', 'observation'], styLst=styLst, cLst='br') axP[k].set_title(title) else: styLst = '-*' shortName = codePdf.loc[var]['shortName'] title = shortName + ' ' + tStr axplot.plotTS(axP[k], t, dfPred[var], tBar=tBar, legLst=['LSTM-sequence'], styLst='-', cLst='b') axplot.plotTS(axP[k], tp, yp, legLst=['LSTM-sample'], styLst='*', cLst='g') axplot.plotTS(axP[k], t, dfObs[var], legLst=['observation'], styLst='*', cLst='r') axP[k].set_title(title)
def funcP(axP, iP, iM): siteNo = siteNoCode[iP] tBar = np.datetime64('2010-01-01') sd = np.datetime64('1980-01-01') for k, code in enumerate(codeM): v1 = dictLSTM[siteNo][code].values v2 = dictWRTDS[siteNo][code].values q = dictObs[siteNo]['00060'].values o = dictObs[siteNo][code].values t = dictObs[siteNo].index.values iS = siteNoLst.index(siteNo) ic = codeLst.index(code) legLst = ['LSTM', 'WRTDS', 'Obs'] axplot.plotTS(axP[k*2], t, [v1, v2, o], tBar=tBar, sd=sd, styLst='--*', cLst='rbk', legLst=legLst) axP[k*2].set_title('site {} corrLSTM={:.2f} corrWRTDS={:.2f}'.format( siteNo, corrMat[iS, ic, 1], corrMat[iS, ic, 2])) td = dictObs[siteNo].index.dayofyear sc = axP[k*2+1].scatter(np.log(q), o, c=td, cmap='hsv', vmin=0, vmax=365)