def funcPoint(iP, axP):
siteNo = siteNoLst[iP]
dfP1, dfObs = basins.loadSeq(outName, siteNo)
rmse1, corr1 = waterQuality.calErrSeq(dfP1[code], dfObs[code])
dfP3 = wqLinear.loadSeq(siteNo, code, 'LR', optT='Y8090')
rmse3, corr3 = waterQuality.calErrSeq(dfP3[code], dfObs[code])
t = dfObs.index.values
tBar = np.datetime64('2000-01-01')
styLst = '-*'
# styLst = ['-', '-*']
dfPLst = [dfP1, dfP3]
rmseLst = [rmse1, rmse3]
corrLst = [corr1, corr3]
for k, dfP in enumerate(dfPLst):
axplot.plotTS(axP[k],
t, [dfP[code], dfObs[code]],
tBar=tBar,
legLst=[modLst[k], 'obs'],
styLst=styLst,
cLst='br')
# ind = np.where(~np.isnan(dfObs[code].values))
# axplot.plotTS(axP[k], t[ind], dfObs[code].values[ind], tBar=tBar,
# legLst=[modLst[k], 'obs'], styLst=styLst, cLst='r')
tStr = '{}, rmse [{:.2f} {:.2f}], corr [{:.2f} {:.2f}]'.format(
siteNo, rmseLst[k][0], rmseLst[k][1], corrLst[k][0], corrLst[k][1])
axP[k].set_title(tStr)
def funcPoint(iP, axP):
siteNo = siteNoLst[iP]
dfP1, dfObs = basins.loadSeq(outName, siteNo)
t = dfPred.index.values.astype(np.datetime64)
tBar = np.datetime64('2000-01-01')
# plot Q
rmse, corr = waterQuality.calErrSeq(dfP1['00060'], dfObs['00060'])
axplot.plotTS(axP[0], t, [dfP1['00060'], dfObs['00060']], tBar=tBar,
legLst=['pred', 'obs'], styLst='--', cLst='br')
tStr = '{}, rmse [{:.2f} {:.2f}], corr [{:.2f} {:.2f}]'.format(
siteNo, rmse[0], rmse[1], corr[0], corr[1])
axP[0].set_title('steamflow '+tStr)
# Silica
rmse, corr = waterQuality.calErrSeq(dfP1[code], dfObs[code])
axplot.plotTS(axP[1], t, [dfP1[code], dfObs[code]], tBar=tBar,
legLst=['LSTM', 'obs'], styLst='-*', cLst='br')
tStr = '{}, rmse [{:.2f} {:.2f}], corr [{:.2f} {:.2f}]'.format(
siteNo, rmse[0], rmse[1], corr[0], corr[1])
axP[1].set_title('Silica '+tStr)
# fourier
df = dfObs[dfObs['00955'].notna().values]
# nt = len(dfObs)
nt = 365*5
x = (df.index.values.astype('datetime64[D]') -
np.datetime64('1979-01-01')).astype(np.float)
y = df['00955'].values
freq = np.fft.fftfreq(nt)[1:]
ls = LombScargle(x, y)
power = ls.power(freq)
df2 = dfP1['00955']
x2 = (df2.index.values.astype('datetime64[D]') -
np.datetime64('1979-01-01')).astype(np.float)
y2 = df2.values
ls2 = LombScargle(x2, y2)
power2 = ls2.power(freq)
axP[2].set_ylabel('normalize spectrum')
indF = np.where(freq > 0)[0]
axP[2].plot(1/freq[indF], power2[indF], 'b', label='lstm')
axP[2].plot(1/freq[indF], power[indF], 'r', label='obs')
axP[2].legend()
axP[3].set_ylabel('power')
axP[2].set_xlabel('period (day)')
axP[3].plot(np.log(freq), np.log(power), '-*')
axP[3].set_xlabel('log(freq)')
axP[3].set_ylabel('log(power)')
def funcPoint(iP, axP):
siteNo = siteNoLst[iP]
dfPred, dfObs = basins.loadSeq(outName, siteNo)
t = dfPred.index.values.astype(np.datetime64)
tBar = np.datetime64('2000-01-01')
info1 = wqData.subsetInfo(trainset)
info2 = wqData.subsetInfo(testset)
ind1 = info1[info1['siteNo'] == siteNo].index
ind2 = info2[info2['siteNo'] == siteNo].index
t1 = info1['date'][ind1].values.astype(np.datetime64)
t2 = info2['date'][ind2].values.astype(np.datetime64)
tp = np.concatenate([t1, t2])
yp = np.concatenate([ycP1[ind1], ycP2[ind2]])
for k, var in enumerate(plotVar):
rmse, corr = waterQuality.calErrSeq(dfPred[var], dfObs[var])
tStr = '{}, rmse [{:.2f} {:.2f}], corr [{:.2f} {:.2f}]'.format(
siteNo, rmse[0], rmse[1], corr[0], corr[1])
if var == '00060':
styLst = '--'
title = 'streamflow ' + tStr
axplot.plotTS(axP[k],
t, [dfPred[var], dfObs[var]],
tBar=tBar,
legLst=['LSTM', 'observation'],
styLst=styLst,
cLst='br')
axP[k].set_title(title)
else:
styLst = '-*'
shortName = codePdf.loc[var]['shortName']
title = shortName + ' ' + tStr
axplot.plotTS(axP[k],
t,
dfPred[var],
tBar=tBar,
legLst=['LSTM-sequence'],
styLst='-',
cLst='b')
axplot.plotTS(axP[k],
tp,
yp,
legLst=['LSTM-sample'],
styLst='*',
cLst='g')
axplot.plotTS(axP[k],
t,
dfObs[var],
legLst=['observation'],
styLst='*',
cLst='r')
axP[k].set_title(title)
def funcPoint(iP, axP):
siteNo = siteNoLst[iP]
dfP1, dfObs = basins.loadSeq(outName, siteNo)
rmse1, corr1 = waterQuality.calErrSeq(dfP1[code], dfObs[code])
t = dfObs.index.values
tBar = np.datetime64('2000-01-01')
axplot.plotTS(axP,
t, [dfP1[code], dfObs[code]],
tBar=tBar,
legLst=['LSTM', 'obs'],
styLst='-*',
cLst='br')
tStr = '{}, rmse [{:.2f} {:.2f}], corr [{:.2f} {:.2f}]'.format(
siteNo, rmse1[0], rmse1[1], corr1[0], corr1[1])
axP.set_title(tStr)
def funcPoint(iP, axP):
siteNo = siteNoLst[iP]
dfPred, dfObs = basins.loadSeq(outName, siteNo)
t = dfPred.index.values.astype(np.datetime64)
tBar = np.datetime64('2000-01-01')
for k, var in enumerate(plotVar):
rmse, corr = waterQuality.calErrSeq(dfPred[var], dfObs[var])
tStr = '{}, rmse [{:.2f} {:.2f}], corr [{:.2f} {:.2f}]'.format(
siteNo, rmse[0], rmse[1], corr[0], corr[1])
if var == '00060':
styLst = '--'
title = 'streamflow ' + tStr
else:
styLst = '-*'
shortName = codePdf.loc[var]['shortName']
title = shortName + ' ' + tStr
axplot.plotTS(axP[k],
t, [dfPred[var], dfObs[var]],
tBar=tBar,
legLst=['pred', 'obs'],
styLst=styLst,
cLst='br')
axP[k].set_title(tStr)
def funcPoint(iP, axP):
siteNo = siteNoLst[iP]
dfPLst = list()
rmseLst = list()
corrLst = list()
for ep in epLst:
dfP, dfObs = basins.loadSeq(outName, siteNo, ep=ep)
rmse, corr = waterQuality.calErrSeq(dfP[code], dfObs[code])
dfPLst.append(dfP)
rmseLst.append(rmse)
corrLst.append(corr)
t = dfObs.index.values
tBar = np.datetime64('2000-01-01')
for k, dfP in enumerate(dfPLst):
axplot.plotTS(axP[k],
t, [dfP[code], dfObs[code]],
tBar=tBar,
legLst=[epLst[k], 'obs'],
styLst='-*',
cLst='br')
tStr = '{}, rmse [{:.2f} {:.2f}], corr [{:.2f} {:.2f}]'.format(
siteNo, rmseLst[k][0], rmseLst[k][1], corrLst[k][0], corrLst[k][1])
axP[k].set_title(tStr)
# trainset = 'Y0010'
# testset = 'Y8090'
optT = trainset
master = basins.loadMaster(outName)
# seq test
siteNoLst = wqData.info['siteNo'].unique().tolist()
basins.testModelSeq(outName, siteNoLst, wqData=wqData)
ns = len(siteNoLst)
# calculate error from sequence
rmseMat = np.ndarray([ns, 2])
corrMat = np.ndarray([ns, 2])
for k, siteNo in enumerate(siteNoLst):
print(k, siteNo)
dfPred, dfObs = basins.loadSeq(outName, siteNo)
rmseLSTM, corrLSTM = waterQuality.calErrSeq(dfPred[code], dfObs[code])
rmseMat[k, :] = rmseLSTM
corrMat[k, :] = corrLSTM
# time series map
dfCrd = gageII.readData(varLst=['LAT_GAGE', 'LNG_GAGE'], siteNoLst=siteNoLst)
lat = dfCrd['LAT_GAGE'].values
lon = dfCrd['LNG_GAGE'].values
codePdf = usgs.codePdf
def funcMap():
figM, axM = plt.subplots(2, 1, figsize=(8, 6))
axplot.mapPoint(axM[0], lat, lon, corrMat[:, 0] - corrMat[:, 1], s=12)
axplot.mapPoint(axM[1], lat, lon, corrMat[:, 1], s=12)
figP, axP = plt.subplots(2, 1, figsize=(8, 6))
code = '00955'
trainset = 'Y8090'
testset = 'Y0010'
master = basins.loadMaster(outName)
# seq test
siteNoLst = wqData.info['siteNo'].unique().tolist()
basins.testModelSeq(outName, siteNoLst, wqData=wqData)
ns = len(siteNoLst)
# calculate error from sequence
rmseMat = np.ndarray([ns, 3, 2])
corrMat = np.ndarray([ns, 3, 2])
for k, siteNo in enumerate(siteNoLst):
print(k, siteNo)
dfPred, dfObs = basins.loadSeq(outName, siteNo)
rmseLSTM, corrLSTM = waterQuality.calErrSeq(dfPred[code], dfObs[code])
dfP2 = wqLinear.loadSeq(siteNo,
code,
'ARMA',
optT='Y8090',
order=(5, 0, 0))
rmseARMA, corrARMA = waterQuality.calErrSeq(dfP2[code], dfObs[code])
dfP3 = wqLinear.loadSeq(siteNo, code, 'LR', optT='Y8090')
rmseLR, corrLR = waterQuality.calErrSeq(dfP3[code], dfObs[code])
rmseMat[k, :, :] = [rmseLSTM, rmseARMA, rmseLR]
corrMat[k, :, :] = [corrLSTM, corrARMA, corrLR]
# box
for (errMat, title) in zip([rmseMat, corrMat], ['RMSE', 'Correlation']):
dataBox = list()
for k in range(2):
outName = 'Silica64Seq-Y8090'
siteNoLst = wqData.siteNoLst
basins2.testModelSeq(outName, siteNoLst, wqData=wqData)
outLst = ['Silica64Seq-Y8090', 'Silica64-Y8090-00955-opt1']
code = '00955'
wqData = waterQuality.DataModelWQ('Silica64')
siteNoLst = wqData.siteNoLst
ns = len(siteNoLst)
rmseMat = np.ndarray([ns, 2, 2])
corrMat = np.ndarray([ns, 2, 2])
for k, siteNo in enumerate(siteNoLst):
for i, out in enumerate(outLst):
print(k, siteNo)
dfP, dfO = basins.loadSeq(out, siteNo)
rmse, corr = waterQuality.calErrSeq(dfP[code], dfO[code])
rmseMat[k, i, :] = rmse
corrMat[k, i, :] = corr
# box
for (errMat, title) in zip([rmseMat, corrMat], ['RMSE', 'Correlation']):
dataBox = list()
for k in range(2):
temp = [errMat[:, i, k] for i in range(2)]
dataBox.append(temp)
label1 = ['B2000', 'A2000']
label2 = ['seq', 'point']
fig = figplot.boxPlot(dataBox, label1=label1, label2=label2, sharey=True)
fig.suptitle(title)
fig.show()
import torch
import os
import json
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
siteNo = '01674500'
code = '00955'
outName = 'Silica64-Y8090-00955-opt1'
dfP1, dfObs = basins.loadSeq(outName, siteNo)
dfP2 = wqLinear.loadSeq(siteNo, code, 'ARMA', optT='Y8090', order=(5, 0, 0))
rmse2, corr2 = waterQuality.calErrSeq(dfP2[code], dfObs[code])
t = dfObs.index.values
tBar = np.datetime64('2000-01-01')
styLst = '-*'
figP, axP = plt.subplots(1, 1, figsize=(8, 6))
axplot.plotTS(axP,
t, [dfP2[code], dfObs[code]],
tBar=tBar,
styLst='-*',
cLst='br')
figP.show()
dfO = dfObs[code]
dfP = dfP2[code]
