# master.runTrain(masterDict, cudaID=cid % 3, screen='test-DA' + str(nDay))
cid = cid + 1
# test original model
caseLst = ['All-90-95']
nDayLst = [3, 10, 30, 100, 365]
for nDay in nDayLst:
caseLst.append('All-90-95-DA' + str(nDay))
outLst = [os.path.join(pathCamels['Out'], save_path, x) for x in caseLst]
subset = 'All'
tRange = [19950101, 20000101]
predLst = list()
for out in outLst:
df, pred, obs = master.test(out,
tRange=tRange,
subset=subset,
basinnorm=True,
epoch=200)
# pred=np.maximum(pred,0)
predLst.append(pred)
# plot box
statDictLst = [stat.statError(x.squeeze(), obs.squeeze()) for x in predLst]
# keyLst = list(statDictLst[0].keys())
keyLst = ['Bias', 'RMSE', 'NSE']
dataBox = list()
for iS in range(len(keyLst)):
statStr = keyLst[iS]
temp = list()
for k in range(len(statDictLst)):
data = statDictLst[k][statStr]
from hydroDL import pathSMAP, master
import os
from hydroDL.master import default
cDir = os.path.dirname(os.path.abspath(__file__))
cDir = r'/home/kxf227/work/GitHUB/pyRnnSMAP/example/'
# define training options
optData = default.update(
default.optDataSMAP,
rootDB=os.path.join(cDir, 'data'),
subset='CONUSv4f1',
tRange=[20150401, 20160401],
)
optModel = default.optLstm
optLoss = default.optLossSigma
optTrain = default.update(master.default.optTrainSMAP, nEpoch=5, saveEpoch=5)
out = os.path.join(cDir, 'output', 'CONUSv4f1_sigma')
masterDict = master.wrapMaster(out, optData, optModel, optLoss, optTrain)
# train
master.train(masterDict)
# test
pred = master.test(out, tRange=[20160401, 20170401], subset='CONUSv4f1')
import os
from hydroDL.data import dbCsv
from hydroDL.post import plot, stat
from hydroDL import master
cDir = os.path.dirname(os.path.abspath(__file__))
out = os.path.join(cDir, 'output', 'CONUSv4f1')
rootDB = os.path.join(cDir, 'data')
nEpoch = 100
tRange = [20160401, 20170401]
# load data
df, yp, yt = master.test(
out, tRange=[20160401, 20170401], subset='CONUSv4f1', epoch=100, reTest=True)
yp = yp.squeeze()
yt = yt.squeeze()
# calculate stat
statErr = stat.statError(yp, yt)
dataGrid = [statErr['RMSE'], statErr['Corr']]
dataTs = [yp, yt]
t = df.getT()
crd = df.getGeo()
mapNameLst = ['RMSE', 'Correlation']
tsNameLst = ['LSTM', 'SMAP']
# plot map and time series
plot.plotTsMap(
dataGrid,
dataTs,
# k = [7, 8, 13]
# %% load data and stat
kcLst = [7, 8, 13]
tRange = [20160401, 20180401]
statLst = list()
statRefLst = list()
for kc in kcLst:
tempLst = list()
for k in range(1, 18):
testName = subsetLst[kc - 1]
if k != kc:
outName = 'ecoRegion{:02d}{:02d}_v2f1_Forcing'.format(kc, k)
else:
outName = 'ecoRegion{:02d}_v2f1_Forcing'.format(kc)
out = os.path.join(pathSMAP['Out_L3_NA'], 'ecoRegion', outName)
df, yp, yt = master.test(out, tRange=tRange, subset=testName)
temp = stat.statError(yp[:, :, 0], yt[:, :, 0])
tempLst.append(temp)
if k == kc:
statRefLst.append(temp)
statLst.append(tempLst)
# %% plot box
keyLst = stat.keyLst
ecoLst = ['{:02d}'.format(x) for x in range(1, 18)]
caseLst = ['{:02d}'.format(x) for x in [7, 8, 13]]
for k in range(len(caseLst)):
dataBox = list()
key = 'RMSE'
for ii in range(len(ecoLst)):
from hydroDL import pathSMAP, master, utils
from hydroDL.master import default
from hydroDL.post import plot, stat
import os
import matplotlib.pyplot as plt
import numpy as np
import torch
# train
optData = default.update(default.optDataSMAP,
rootDB=pathSMAP['DB_L3_NA'],
subset='CONUSv4f1',
tRange=[20150402, 20180401],
daObs=1)
optModel = default.optLstmClose
optLoss = default.optLossRMSE
optTrain = default.update(default.optTrainSMAP, nEpoch=500)
out = os.path.join(pathSMAP['Out_L3_NA'], 'DA', 'CONUS_DA_3yr')
masterDict = master.wrapMaster(out, optData, optModel, optLoss, optTrain)
master.runTrain(masterDict, cudaID=2, screen='DA')
# test
subset = 'CONUS'
tRange = [20150402, 20180401]
# out = os.path.join(pathSMAP['Out_L3_NA'], 'DA', 'CONUSv2f1_DA2015')
out = os.path.join(pathSMAP['Out_L3_NA'], 'DA', 'CONUS_DA_3yr')
df, yf, obs = master.test(out, tRange=tRange, subset=subset, batchSize=100)
# /home/kxf227/work/GitHUB/hydroDL-dev/app/closeLoop/fullCONUS.py
dataPred[:, :, k] = pd.read_csv(filePath,
dtype=np.float,
header=None).values
# transform back to the original observation
temppred = camels.transNormbyDic(dataPred,
"runoff",
statDict,
toNorm=False)
pred = camels.basinNorm(temppred, subset, toNorm=False)
elif interfaceOpt == 0: # only for models trained by the pro interface
df, pred, obs = master.test(
out,
tRange=tRange,
subset=subset,
batchSize=testBatch,
basinnorm=True,
epoch=TestEPOCH,
reTest=True,
)
# change the units ft3/s to m3/s
obs = obs * 0.0283168
pred = pred * 0.0283168
predLst.append(pred) # the prediction list for all the models
# calculate statistic metrics
statDictLst = [stat.statError(x.squeeze(), obs.squeeze()) for x in predLst]
# Show boxplots of the results
plt.rcParams["font.size"] = 14
temppred = camels.transNormbyDic(dataPred,
"runoff",
statDict,
toNorm=False)
pred = camels.basinNorm(temppred,
np.array(testBasin),
toNorm=False)
elif interfaceOpt == 0:
if migOptLst[icount] is True:
# for FDC migration case
df, pred, obs = master.test(
out,
tRange=tRange,
subset=testBasin,
basinnorm=True,
epoch=testEpoch,
reTest=True,
FDCgage=FDCList[imig],
)
imig = imig + 1
else:
# for other ordinary cases
df, pred, obs = master.test(
out,
tRange=tRange,
subset=testBasin,
basinnorm=True,
epoch=testEpoch,
reTest=True,
)
outLst = [os.path.join(rootOut, save_path, x) for x in caseLst]
subset = 'All' # 'All': use all the CAMELS gages to test; Or pass the gage list
tRange = [20141001, 20161001] # Testing period
predLst = list()
obsLst = list()
predLst_res = list()
obsLst_res = list()
statDictLst = []
for i, out in enumerate(outLst):
#df, pred, obs = master.test(out, TempTarget, forcing_path[i], attr_path[i], tRange=tRange, subset=subset, basinnorm=True, epoch=TestEPOCH, reTest=True)
df, pred, obs, x = master.test(out,
TempTarget,
forcing_path,
attr_path,
tRange=tRange,
subset=subset,
basinnorm=False,
epoch=TestEPOCH,
reTest=True)
# change the units ft3/s to m3/s
#obs = obs * 0.0283168
#pred = pred * 0.0283168
predLst.append(pred) # the prediction list for all the models
obsLst.append(obs)
np.save(os.path.join(out, 'pred.npy'), pred)
np.save(os.path.join(out, 'obs.npy'), obs)
f = np.load(
os.path.join(out, 'x.npy')
) # it has been saved previously in the out directory (forcings)
from hydroDL import pathSMAP, master
import os
optData = master.updateOpt(
master.default.optDataCsv,
path=pathSMAP['DB_L3_NA'],
subset='CONUSv4f1',
dateRange=[20150401, 20160331])
optModel = master.default.optLstm
optLoss = master.default.optLoss
optTrain = master.default.optTrainSMAP
out = os.path.join(pathSMAP['Out_L3_Global'], 'regTest')
masterDict = master.wrapMaster(out, optData, optModel, optLoss, optTrain)
# master.train(masterDict, overwrite=True)
pred = master.test(
out, tRange=[20160401, 20170331], subset='CONUSv4f1', epoch=400)
optTrain = master.default.optTrainSMAP
out = os.path.join(pathSMAP['Out_L3_NA'], 'ecoRegion',
subsetLst[k] + '_' + case)
masterDict = master.wrapMaster(out, optData, optModel, optLoss, optTrain)
master.runTrain(masterDict, cudaID=cid % 3, screen=subsetLst[k])
cid = cid + 1
# master.train(masterDict)
# test
ss = ''
for k in range(len(subsetLst)):
for case in caseLst:
out = os.path.join(pathSMAP['Out_L3_NA'], 'ecoRegion',
subsetLst[k] + '_' + case)
try:
master.test(out, tRange=[20160401, 20180401], subset=subsetLst[k])
except:
ss = ss + 'ecoRegion ' + str(k) + ' case ' + case + '; '
# retrain some models
# rtEcoLst = [1, 2, 5, 7, 8, 10, 13, 16, 17]
# rtCaseLst = [
# 'Forcing', 'Soilm', 'Soilm', 'Forcing', 'Soilm', 'Forcing', 'Forcing',
# 'Forcing', 'Soilm'
# ]
# rtEcoLst = [1, 4, 7, 16]
# rtCaseLst = ['Soilm', 'Soilm', 'Soilm', 'Soilm']
# cid = 2
# for kk in range(len(rtEcoLst)):
# k = rtEcoLst[kk] - 1
# case = rtCaseLst[kk]
doLst.append('test')
doLst.append('post')
saveDir = os.path.join(pathSMAP['dirResult'], 'DA', 'paper')
# test
# torch.cuda.set_device(2)
subset = 'CONUSv2f1'
tRange = [20160501, 20161001]
outLst = [
'CONUSv2f1_DA2015', 'CONUSv2f1_DA_2015RK', 'CONUSv2f1_DA_Prcp_2015',
'CONUSv2f1_DA_Prcp_2015RK'
]
yfLst = list()
for outName in outLst:
out = os.path.join(pathSMAP['Out_L3_NA'], 'DA', outName)
df, yf, obs = master.test(out, tRange=tRange, subset=subset)
yfLst.append(yf.squeeze())
obs = obs.squeeze()
# figure out how many days observation lead
# maskObs = 1 * ~np.isnan(obs.squeeze()) * (qualSuc.squeeze() == 0)
maskObs = 1 * ~np.isnan(obs.squeeze())
maskDay = np.zeros(maskObs.shape).astype(int)
ngrid, nt = maskObs.shape
for j in range(ngrid):
temp = 0
for i in range(nt):
maskDay[j, i] = temp
if maskObs[j, i] == 1:
temp = 1
else:
caseLst2 = ['Forcing', 'Soilm']
yrLst = [20160401, 20170401]
statLst = list()
for k in range(len(subsetLst)):
testName = subsetLst[k]
tempLst = list()
for case1 in caseLst1:
for case2 in caseLst2:
if case1 == 'Local':
outName = outLst[k] + '_' + case2
elif case1 == 'Global':
outName = 'Global_v4f1_y1' + '_' + case2
out = os.path.join(pathSMAP['Out_L3_Global'], outName)
df, yp, yt = master.test(out,
tRange=yrLst,
subset=testName,
epoch=500)
temp = stat.statError(yp[:, :, 0], yt[:, :, 0])
tempLst.append(temp)
statLst.append(tempLst)
# plot box
keyLst = stat.keyLst
caseLst = list()
for case1 in caseLst1:
for case2 in caseLst2:
caseLst.append(case1 + ' ' + case2)
for k in range(len(keyLst)):
dataBox = list()
key = keyLst[k]
rootDB=pathSMAP['DB_L3_NA'],
target=['SMAP_AM', 'SOILM_0-10_NOAH'],
subset='CONUSv4f1',
tRange=[20150401, 20160401])
optModel = default.optLstm
optLoss = default.optLossSigma
optTrain = default.update(default.optTrainSMAP, nEpoch=100)
out = os.path.join(cDir, 'output', 'CONUSv4f1_multi')
masterDict = wrapMaster(out, optData, optModel, optLoss, optTrain)
# train
# train(masterDict)
# runTrain(masterDict, cudaID=2, screen='LSTM-multi')
# test
df, yp, yt, sigma = test(out, tRange=[20160401, 20170401], subset='CONUSv4f1')
# plot ts MAP
dataGrid = list()
dataTs = list()
for k in range(2):
statErr = stat.statError(yp[:, :, k], yt[:, :, k])
dataGrid.append(statErr['RMSE'])
dataTs.append([yp[:, :, k], yt[:, :, k]])
t = df.getT()
crd = df.getGeo()
mapNameLst = ['RMSE ', 'RMSE']
tsNameLst = ['LSTM', 'SMAP']
plot.plotTsMap(dataGrid,
dataTs,
lat=crd[0],
cDir = os.path.dirname(os.path.abspath(__file__))
cDir = r'/home/kxf227/work/GitHUB/pyRnnSMAP/example/'
rootDB = os.path.join(cDir, 'data')
nEpoch = 500
out = os.path.join(cDir, 'output', 'CONUSv4f1')
tRange = [20160401, 20170401]
# load data
df = dbCsv.DataframeCsv(
rootDB=rootDB, subset='CONUSv4f1', tRange=tRange)
yt = df.getData(varT='SMAP_AM', doNorm=False, rmNan=False)
yt = yt.squeeze()
yp = master.test(
out, tRange=[20160401, 20170401], subset='CONUSv4f1', epoch=500)
yp = yp.squeeze()
# calculate stat
statErr = stat.statError(yp, yt)
dataGrid = [statErr['RMSE'], statErr['Corr']]
dataTs = [yp, yt]
t = df.getT()
crd = df.getGeo()
mapNameLst = ['RMSE', 'Correlation']
tsNameLst = ['LSTM', 'SMAP']
colorMap = None
colorTs = None
# plot map and time series
plot.plotTsMap(
from hydroDL.post import axplot, stat, figplot
import os
import matplotlib.pyplot as plt
import numpy as np
import torch
import matplotlib
matplotlib.rcParams.update({'font.size': 14})
matplotlib.rcParams.update({'lines.linewidth': 2})
matplotlib.rcParams.update({'lines.markersize': 6})
# test
subset = 'CONUSv2f1'
out = os.path.join(pathSMAP['Out_L3_NA'], 'DA', 'CONUSv2f1_DA2015')
df1, yf1, obs1 = master.test(out,
tRange=[20150402, 20180401],
subset=subset,
batchSize=100)
df2, yf2, obs2 = master.test(out,
tRange=[20160401, 20180401],
subset=subset,
batchSize=100)
out = os.path.join(pathSMAP['Out_L3_NA'], 'DA', 'CONUSv2f1_LSTM2015')
df1, yp1, obs1 = master.test(out,
tRange=[20150402, 20180401],
subset=subset,
batchSize=100)
df2, yp2, obs2 = master.test(out,
tRange=[20160401, 20180401],
subset=subset,
batchSize=100)
out = os.path.join(pathSMAP['Out_L3_NA'], 'DA', 'CONUSv2f1')
masterDict = master.wrapMaster(out, optData, optModel, optLoss, optTrain)
master.runTrain(masterDict, cudaID=0, screen='LSTM')
# test
if 'test' in doLst:
torch.cuda.set_device(2)
outLst = [
os.path.join(pathSMAP['Out_L3_NA'], 'DA', 'CONUSv2f1_d' + str(nd))
for nd in dLst
]
subset = 'CONUSv2f1'
tRange = [20160501, 20170501]
predLst = list()
outLSTM = os.path.join(pathSMAP['Out_L3_NA'], 'DA', 'CONUSv2f1')
df, pred, obs = master.test(
outLSTM, tRange=tRange, subset=subset, batchSize=100)
predLst.append(pred.squeeze())
for out in outLst:
df, pred, obs = master.test(
out, tRange=tRange, subset=subset, batchSize=100)
predLst.append(pred.squeeze())
obs = obs.squeeze()
# plot box - latency
# if 'post' in doLst:
caseLst = ['Predict'] + ['Nowcast ' + str(nd) + 'd latency' for nd in dLst]
statLst1 = [stat.statError(x, obs) for x in predLst]
keyLst = list(statLst1[0].keys())
dataBox = list()
for iS in range(len(keyLst)):
statStr = keyLst[iS]