if 'test' in doLst:
# load data
df = hydroDL.data.dbCsv.DataframeCsv(rootDB=rootDB,
subset='CONUSv4f1',
tRange=ty2)
x = df.getDataTs(dbCsv.varForcing, doNorm=True, rmNan=True)
c = df.getDataConst(dbCsv.varConst, doNorm=True, rmNan=True)
nx = x.shape[-1] + c.shape[-1]
yT = df.getDataTs('SMAP_AM', doNorm=False, rmNan=False)
yT = yT[:, :, 0]
# test
ypLstmLst = list()
ypAnnLst = list()
modelName = 'LSTM'
model = train.loadModel(outFolder, 100, modelName=modelName)
yp = train.testModel(model, x, c, batchSize=100).squeeze()
ypLstmLst.append(
dbCsv.transNorm(yp, rootDB=rootDB, fieldName='SMAP_AM', fromRaw=False))
for k in dLst:
sd = utils.time.t2dt(ty2[0]) - dt.timedelta(days=k)
ed = utils.time.t2dt(ty2[1]) - dt.timedelta(days=k)
df2 = hydroDL.data.dbCsv.DataframeCsv(rootDB=rootDB,
subset='CONUSv4f1',
tRange=[sd, ed])
obs = df2.getDataTs('SMAP_AM', doNorm=True, rmNan=False)
modelName = 'LSTM-DA-' + str(k)
model = train.loadModel(outFolder, nEpoch, modelName=modelName)
yP = train.testModel(model, (x, obs), c, batchSize=100).squeeze()
ypLstmLst.append(
df2 = camels.DataframeCamels(subset='all', tRange=[20050101, 20150101])
x2 = df2.getDataTS(varLst=camels.forcingLst, doNorm=True, rmNan=True)
c2 = df2.getDataConst(varLst=camels.attrLstSel, doNorm=True, rmNan=True)
yt2 = df2.getDataObs(doNorm=False, rmNan=False).squeeze()
dfz1 = camels.DataframeCamels(subset='all', tRange=[20041231, 20141231])
z1 = dfz1.getDataObs(doNorm=True, rmNan=True)
# z1 = interp.interpNan1d(z1, mode='pre')
xz1 = np.concatenate([x2, z1], axis=2)
dfz2 = camels.DataframeCamels(subset='all', tRange=[20041225, 20141225])
z2 = dfz2.getDataObs(doNorm=True, rmNan=True)
# z2 = interp.interpNan1d(z2, mode='pre')
xz2 = np.concatenate([x2, z2], axis=2)
model1 = train.loadModel(outFolder, nEpoch, modelName='LSTM')
yp1 = train.testModel(model1, x2, c2)
yp1 = camels.transNorm(yp1, 'usgsFlow', toNorm=False).squeeze()
model2 = train.loadModel(outFolder, nEpoch, modelName='DA-1')
yp2 = train.testModel(model2, xz1, c2)
yp2 = camels.transNorm(yp2, 'usgsFlow', toNorm=False).squeeze()
model3 = train.loadModel(outFolder, nEpoch, modelName='DA-7')
yp3 = train.testModel(model3, xz2, c2)
yp3 = camels.transNorm(yp3, 'usgsFlow', toNorm=False).squeeze()
yLst = [yt2, yp1, yp2, yp3]
# plot box
statDictLst = [
ypLst = list()
df = hydroDL.data.dbCsv.DataframeCsv(rootDB=rootDB,
subset='CONUSv4f1',
tRange=ty2)
yT = df.getData(varT='SMAP_AM', doNorm=False, rmNan=False).squeeze()
if 'testLstm' in doLst:
df = hydroDL.data.dbCsv.DataframeCsv(rootDB=rootDB,
subset='CONUSv4f1',
tRange=ty2)
x = df.getData(varT=dbCsv.varForcing,
varC=dbCsv.varConst,
doNorm=True,
rmNan=True)
model = train.loadModel(outFolder, nEpoch, modelName='lstm_y1')
yP = train.testModel(model, x).squeeze()
ypLst.append(
dbCsv.transNorm(yP, rootDB=rootDB, fieldName='SMAP_AM', fromRaw=False))
if 'testCnn' in doLst:
dfc = hydroDL.data.dbCsv.DataframeCsv(rootDB=rootDB,
subset='CONUSv4f1',
tRange=ty1)
xc = dfc.getData(varT=dbCsv.varForcing,
varC=dbCsv.varConst,
doNorm=True,
rmNan=True)
yc = dfc.getData(varT='SMAP_AM', doNorm=True, rmNan=False)
yc[:, :, 0] = utils.interpNan(yc[:, :, 0])
z = np.concatenate((yc, xc), axis=2)
df = hydroDL.data.dbCsv.DataframeCsv(rootDB=rootDB,
model = train.trainModel(
model, (x, yc), y, lossFun, nEpoch=nEpoch, miniBatch=[100, 60])
modelName = 'cnnForcast' + str(opt)
train.saveModel(outFolder, model, nEpoch, modelName=modelName)
ypLst = list()
df = hydroDL.data.dbCsv.DataframeCsv(
rootDB=rootDB, subset='CONUSv4f1', tRange=ty2)
yT = df.getData(varT='SMAP_AM', doNorm=False, rmNan=False).squeeze()
if 'testLstm' in doLst:
df = hydroDL.data.dbCsv.DataframeCsv(
rootDB=rootDB, subset='CONUSv4f1', tRange=ty2)
x = df.getData(
varT=dbCsv.varForcing, varC=dbCsv.varConst, doNorm=True, rmNan=True)
model = train.loadModel(outFolder, nEpoch, modelName='lstmForcast')
yP = train.testModel(model, x).squeeze()
ypLst.append(
dbCsv.transNorm(yP, rootDB=rootDB, fieldName='SMAP_AM', fromRaw=False))
if 'testCnn' in doLst:
df = hydroDL.data.dbCsv.DataframeCsv(
rootDB=rootDB, subset='CONUSv4f1', tRange=tyc2)
x = df.getData(
varT=dbCsv.varForcing, varC=dbCsv.varConst, doNorm=True, rmNan=True)
y = df.getData(varT='SMAP_AM', doNorm=True, rmNan=False)
yc = np.copy(y)
yc[:, :, 0] = utils.interpNan(yc[:, :, 0], mode='pre')
for opt in range(1, 3):
modelName = 'cnnForcast' + str(opt)
y1 = df1.getDataObs(doNorm=True, rmNan=False) c1 = df1.getDataConst(varLst=camels.attrLstSel, doNorm=True, rmNan=True) yt1 = df1.getDataObs(doNorm=False, rmNan=False).squeeze() dfz1 = camels.DataframeCsv(subset='all', tRange=[20141231, 20091231]) z1 = dfz1.getDataObs(doNorm=True, rmNan=False) dfz2 = camels.DataframeCsv(subset='all', tRange=[20141227, 20091227]) z2 = dfz2.getDataObs(doNorm=True, rmNan=False) df2 = camels.DataframeCsv(subset='all', tRange=[20100101, 20150101]) x2 = df2.getDataTS(varLst=camels.forcingLst, doNorm=True, rmNan=True) c2 = df2.getDataConst(varLst=camels.attrLstSel, doNorm=True, rmNan=True) yt2 = df2.getDataObs(doNorm=False, rmNan=False).squeeze() model = train.loadModel(outFolder, 100, modelName='test') yp1 = train.testModel(model, x1, c1) yp1 = camels.transNorm(yp1, 'usgsFlow', toNorm=False).squeeze() yp2 = train.testModel(model, x2, c2) yp2 = camels.transNorm(yp2, 'usgsFlow', toNorm=False).squeeze() statErr1 = stat.statError(yp1, yt2) statErr2 = stat.statError(yp2, yt2) dataMap = [statErr2['Corr'], statErr1['Corr'] - statErr2['Corr']] dataTs = [yt2, yp2] t = df2.getT() crd = df2.getGeo() mapNameLst = ['Test Corr', 'Train Corr - Test Corr'] tsNameLst = ['USGS', 'LSTM'] colorMap = None colorTs = None