# load data
df = hydroDL.data.dbCsv.DataframeCsv(rootDB=rootDB,
subset='CONUSv4f1',
tRange=ty2)
x = df.getDataTs(dbCsv.varForcing, doNorm=True, rmNan=True)
c = df.getDataConst(dbCsv.varConst, doNorm=True, rmNan=True)
nx = x.shape[-1] + c.shape[-1]
yT = df.getDataTs('SMAP_AM', doNorm=False, rmNan=False)
yT = yT[:, :, 0]
# test
ypLstmLst = list()
ypAnnLst = list()
modelName = 'LSTM'
model = train.loadModel(outFolder, 100, modelName=modelName)
yp = train.testModel(model, x, c, batchSize=100).squeeze()
ypLstmLst.append(
dbCsv.transNorm(yp, rootDB=rootDB, fieldName='SMAP_AM', fromRaw=False))
for k in dLst:
sd = utils.time.t2dt(ty2[0]) - dt.timedelta(days=k)
ed = utils.time.t2dt(ty2[1]) - dt.timedelta(days=k)
df2 = hydroDL.data.dbCsv.DataframeCsv(rootDB=rootDB,
subset='CONUSv4f1',
tRange=[sd, ed])
obs = df2.getDataTs('SMAP_AM', doNorm=True, rmNan=False)
modelName = 'LSTM-DA-' + str(k)
model = train.loadModel(outFolder, nEpoch, modelName=modelName)
yP = train.testModel(model, (x, obs), c, batchSize=100).squeeze()
ypLstmLst.append(
dbCsv.transNorm(yP,
x2 = df2.getDataTS(varLst=camels.forcingLst, doNorm=True, rmNan=True)
c2 = df2.getDataConst(varLst=camels.attrLstSel, doNorm=True, rmNan=True)
yt2 = df2.getDataObs(doNorm=False, rmNan=False).squeeze()
dfz1 = camels.DataframeCamels(subset='all', tRange=[20041231, 20141231])
z1 = dfz1.getDataObs(doNorm=True, rmNan=True)
# z1 = interp.interpNan1d(z1, mode='pre')
xz1 = np.concatenate([x2, z1], axis=2)
dfz2 = camels.DataframeCamels(subset='all', tRange=[20041225, 20141225])
z2 = dfz2.getDataObs(doNorm=True, rmNan=True)
# z2 = interp.interpNan1d(z2, mode='pre')
xz2 = np.concatenate([x2, z2], axis=2)
model1 = train.loadModel(outFolder, nEpoch, modelName='LSTM')
yp1 = train.testModel(model1, x2, c2)
yp1 = camels.transNorm(yp1, 'usgsFlow', toNorm=False).squeeze()
model2 = train.loadModel(outFolder, nEpoch, modelName='DA-1')
yp2 = train.testModel(model2, xz1, c2)
yp2 = camels.transNorm(yp2, 'usgsFlow', toNorm=False).squeeze()
model3 = train.loadModel(outFolder, nEpoch, modelName='DA-7')
yp3 = train.testModel(model3, xz2, c2)
yp3 = camels.transNorm(yp3, 'usgsFlow', toNorm=False).squeeze()
yLst = [yt2, yp1, yp2, yp3]
# plot box
statDictLst = [
stat.statError(yp1, yt2),
outFolder = os.path.join(hydroDL.pathSMAP['outTest'], 'closeLoop')
ty1 = [20150401, 20160401]
ty2 = [20160401, 20170401]
ty3 = [20170401, 20180401]
df = hydroDL.data.dbCsv.DataframeCsv(
rootDB=rootDB, subset='CONUSv4f1', tRange=ty2)
x = df.getData(
varT=dbCsv.varForcing, varC=dbCsv.varConst, doNorm=True, rmNan=True)
y = df.getData(varT='SMAP_AM', doNorm=True, rmNan=False)
yt = df.getData(varT='SMAP_AM', doNorm=False, rmNan=False)
yt = yt[:, :, 0]
ypLst = list()
modelName = 'LSTM'
model = train.loadModel(outFolder, nEpoch, modelName=modelName)
yp = train.testModel(model, x, batchSize=100).squeeze()
ypLst.append(
dbCsv.transNorm(yp, rootDB=rootDB, fieldName='SMAP_AM', fromRaw=False))
modelName = 'LSTM-DA'
model = train.loadModel(outFolder, nEpoch, modelName=modelName)
yp = train.testModel(model, x, z=y, batchSize=100).squeeze()
ypLst.append(
dbCsv.transNorm(yp, rootDB=rootDB, fieldName='SMAP_AM', fromRaw=False))
##
statErr1 = stat.statError(ypLst[0], yt)
statErr2 = stat.statError(ypLst[1], yt)
dataGrid = [statErr2['RMSE'], statErr2['RMSE'] - statErr1['RMSE']]
dataTs = [ypLst[0], ypLst[1], yt]
t = df.getT()
statDict,
toNorm=True)
dadata = np.squeeze(dadata) # dim Ngrid*Nday
fdcdata = master.master.calFDC(dadata)
print("FDC was calculated and used!")
xIn = (xTest, fdcdata)
else:
xIn = xTest
# load and forward the model for testing
testmodel = loadModel(out, epoch=testEpoch)
filePathLst = master.master.namePred(
out, tRange, "All", epoch=testEpoch
) # prepare the name of csv files to save testing results
train.testModel(testmodel,
xIn,
c=attrs,
filePathLst=filePathLst)
# read out predictions
dataPred = np.ndarray(
[obs.shape[0], obs.shape[1],
len(filePathLst)])
for k in range(len(filePathLst)):
filePath = filePathLst[k]
dataPred[:, :, k] = pd.read_csv(filePath,
dtype=np.float,
header=None).values
# transform back to the original observation
temppred = camels.transNormbyDic(dataPred,
"runoff",
statDict,
toNorm=False)
from hydroDL.model import rnn, crit, train
df1 = hydroDL.data.dbCsv.DataframeCsv(rootDB=hydroDL.pathSMAP['DB_L3_NA'],
subset='CONUSv4f1',
tRange=[20150401, 20160401])
x1 = df1.getData(varT=dbCsv.varForcing,
varC=dbCsv.varConst,
doNorm=True,
rmNan=True)
y1 = df1.getData(varT='SMAP_AM', doNorm=True, rmNan=False)
nx = x1.shape[-1]
ny = 2
model = rnn.CudnnLstmModel(nx=nx, ny=ny, hiddenSize=64)
lossFun = crit.SigmaLoss()
model = hydroDL.model.train.trainModel(model,
x1,
y1,
lossFun,
nEpoch=5,
miniBatch=(30, 100))
df2 = hydroDL.data.dbCsv.DataframeCsv(rootDB=hydroDL.pathSMAP['DB_L3_NA'],
subset='CONUSv4f1',
tRange=[20150401, 20160401])
x2 = df2.getData(varT=dbCsv.varForcing,
varC=dbCsv.varConst,
doNorm=True,
rmNan=True)
y2 = df2.getData(varT='SMAP_AM', doNorm=True, rmNan=False)
yp = train.testModel(model, x2)
model, (x, yc), y, lossFun, nEpoch=nEpoch, miniBatch=[100, 60])
modelName = 'cnnForcast' + str(opt)
train.saveModel(outFolder, model, nEpoch, modelName=modelName)
ypLst = list()
df = hydroDL.data.dbCsv.DataframeCsv(
rootDB=rootDB, subset='CONUSv4f1', tRange=ty2)
yT = df.getData(varT='SMAP_AM', doNorm=False, rmNan=False).squeeze()
if 'testLstm' in doLst:
df = hydroDL.data.dbCsv.DataframeCsv(
rootDB=rootDB, subset='CONUSv4f1', tRange=ty2)
x = df.getData(
varT=dbCsv.varForcing, varC=dbCsv.varConst, doNorm=True, rmNan=True)
model = train.loadModel(outFolder, nEpoch, modelName='lstmForcast')
yP = train.testModel(model, x).squeeze()
ypLst.append(
dbCsv.transNorm(yP, rootDB=rootDB, fieldName='SMAP_AM', fromRaw=False))
if 'testCnn' in doLst:
df = hydroDL.data.dbCsv.DataframeCsv(
rootDB=rootDB, subset='CONUSv4f1', tRange=tyc2)
x = df.getData(
varT=dbCsv.varForcing, varC=dbCsv.varConst, doNorm=True, rmNan=True)
y = df.getData(varT='SMAP_AM', doNorm=True, rmNan=False)
yc = np.copy(y)
yc[:, :, 0] = utils.interpNan(yc[:, :, 0], mode='pre')
for opt in range(1, 3):
modelName = 'cnnForcast' + str(opt)
model = train.loadModel(outFolder, nEpoch, modelName=modelName)
ypLst = list()
df = hydroDL.data.dbCsv.DataframeCsv(rootDB=rootDB,
subset='CONUSv4f1',
tRange=ty2)
yT = df.getData(varT='SMAP_AM', doNorm=False, rmNan=False).squeeze()
if 'testLstm' in doLst:
df = hydroDL.data.dbCsv.DataframeCsv(rootDB=rootDB,
subset='CONUSv4f1',
tRange=ty2)
x = df.getData(varT=dbCsv.varForcing,
varC=dbCsv.varConst,
doNorm=True,
rmNan=True)
model = train.loadModel(outFolder, nEpoch, modelName='lstm_y1')
yP = train.testModel(model, x).squeeze()
ypLst.append(
dbCsv.transNorm(yP, rootDB=rootDB, fieldName='SMAP_AM', fromRaw=False))
if 'testCnn' in doLst:
dfc = hydroDL.data.dbCsv.DataframeCsv(rootDB=rootDB,
subset='CONUSv4f1',
tRange=ty1)
xc = dfc.getData(varT=dbCsv.varForcing,
varC=dbCsv.varConst,
doNorm=True,
rmNan=True)
yc = dfc.getData(varT='SMAP_AM', doNorm=True, rmNan=False)
yc[:, :, 0] = utils.interpNan(yc[:, :, 0])
z = np.concatenate((yc, xc), axis=2)
df = hydroDL.data.dbCsv.DataframeCsv(rootDB=rootDB,
subset='CONUSv4f1',
c1 = df1.getDataConst(varLst=camels.attrLstSel, doNorm=True, rmNan=True) yt1 = df1.getDataObs(doNorm=False, rmNan=False).squeeze() dfz1 = camels.DataframeCsv(subset='all', tRange=[20141231, 20091231]) z1 = dfz1.getDataObs(doNorm=True, rmNan=False) dfz2 = camels.DataframeCsv(subset='all', tRange=[20141227, 20091227]) z2 = dfz2.getDataObs(doNorm=True, rmNan=False) df2 = camels.DataframeCsv(subset='all', tRange=[20100101, 20150101]) x2 = df2.getDataTS(varLst=camels.forcingLst, doNorm=True, rmNan=True) c2 = df2.getDataConst(varLst=camels.attrLstSel, doNorm=True, rmNan=True) yt2 = df2.getDataObs(doNorm=False, rmNan=False).squeeze() model = train.loadModel(outFolder, 100, modelName='test') yp1 = train.testModel(model, x1, c1) yp1 = camels.transNorm(yp1, 'usgsFlow', toNorm=False).squeeze() yp2 = train.testModel(model, x2, c2) yp2 = camels.transNorm(yp2, 'usgsFlow', toNorm=False).squeeze() statErr1 = stat.statError(yp1, yt2) statErr2 = stat.statError(yp2, yt2) dataMap = [statErr2['Corr'], statErr1['Corr'] - statErr2['Corr']] dataTs = [yt2, yp2] t = df2.getT() crd = df2.getGeo() mapNameLst = ['Test Corr', 'Train Corr - Test Corr'] tsNameLst = ['USGS', 'LSTM'] colorMap = None colorTs = None