if 'test' in doLst: # load data df = hydroDL.data.dbCsv.DataframeCsv(rootDB=rootDB, subset='CONUSv4f1', tRange=ty2) x = df.getDataTs(dbCsv.varForcing, doNorm=True, rmNan=True) c = df.getDataConst(dbCsv.varConst, doNorm=True, rmNan=True) nx = x.shape[-1] + c.shape[-1] yT = df.getDataTs('SMAP_AM', doNorm=False, rmNan=False) yT = yT[:, :, 0] # test ypLstmLst = list() ypAnnLst = list() modelName = 'LSTM' model = train.loadModel(outFolder, 100, modelName=modelName) yp = train.testModel(model, x, c, batchSize=100).squeeze() ypLstmLst.append( dbCsv.transNorm(yp, rootDB=rootDB, fieldName='SMAP_AM', fromRaw=False)) for k in dLst: sd = utils.time.t2dt(ty2[0]) - dt.timedelta(days=k) ed = utils.time.t2dt(ty2[1]) - dt.timedelta(days=k) df2 = hydroDL.data.dbCsv.DataframeCsv(rootDB=rootDB, subset='CONUSv4f1', tRange=[sd, ed]) obs = df2.getDataTs('SMAP_AM', doNorm=True, rmNan=False) modelName = 'LSTM-DA-' + str(k) model = train.loadModel(outFolder, nEpoch, modelName=modelName) yP = train.testModel(model, (x, obs), c, batchSize=100).squeeze() ypLstmLst.append(
df2 = camels.DataframeCamels(subset='all', tRange=[20050101, 20150101]) x2 = df2.getDataTS(varLst=camels.forcingLst, doNorm=True, rmNan=True) c2 = df2.getDataConst(varLst=camels.attrLstSel, doNorm=True, rmNan=True) yt2 = df2.getDataObs(doNorm=False, rmNan=False).squeeze() dfz1 = camels.DataframeCamels(subset='all', tRange=[20041231, 20141231]) z1 = dfz1.getDataObs(doNorm=True, rmNan=True) # z1 = interp.interpNan1d(z1, mode='pre') xz1 = np.concatenate([x2, z1], axis=2) dfz2 = camels.DataframeCamels(subset='all', tRange=[20041225, 20141225]) z2 = dfz2.getDataObs(doNorm=True, rmNan=True) # z2 = interp.interpNan1d(z2, mode='pre') xz2 = np.concatenate([x2, z2], axis=2) model1 = train.loadModel(outFolder, nEpoch, modelName='LSTM') yp1 = train.testModel(model1, x2, c2) yp1 = camels.transNorm(yp1, 'usgsFlow', toNorm=False).squeeze() model2 = train.loadModel(outFolder, nEpoch, modelName='DA-1') yp2 = train.testModel(model2, xz1, c2) yp2 = camels.transNorm(yp2, 'usgsFlow', toNorm=False).squeeze() model3 = train.loadModel(outFolder, nEpoch, modelName='DA-7') yp3 = train.testModel(model3, xz2, c2) yp3 = camels.transNorm(yp3, 'usgsFlow', toNorm=False).squeeze() yLst = [yt2, yp1, yp2, yp3] # plot box statDictLst = [
ypLst = list() df = hydroDL.data.dbCsv.DataframeCsv(rootDB=rootDB, subset='CONUSv4f1', tRange=ty2) yT = df.getData(varT='SMAP_AM', doNorm=False, rmNan=False).squeeze() if 'testLstm' in doLst: df = hydroDL.data.dbCsv.DataframeCsv(rootDB=rootDB, subset='CONUSv4f1', tRange=ty2) x = df.getData(varT=dbCsv.varForcing, varC=dbCsv.varConst, doNorm=True, rmNan=True) model = train.loadModel(outFolder, nEpoch, modelName='lstm_y1') yP = train.testModel(model, x).squeeze() ypLst.append( dbCsv.transNorm(yP, rootDB=rootDB, fieldName='SMAP_AM', fromRaw=False)) if 'testCnn' in doLst: dfc = hydroDL.data.dbCsv.DataframeCsv(rootDB=rootDB, subset='CONUSv4f1', tRange=ty1) xc = dfc.getData(varT=dbCsv.varForcing, varC=dbCsv.varConst, doNorm=True, rmNan=True) yc = dfc.getData(varT='SMAP_AM', doNorm=True, rmNan=False) yc[:, :, 0] = utils.interpNan(yc[:, :, 0]) z = np.concatenate((yc, xc), axis=2) df = hydroDL.data.dbCsv.DataframeCsv(rootDB=rootDB,
model = train.trainModel( model, (x, yc), y, lossFun, nEpoch=nEpoch, miniBatch=[100, 60]) modelName = 'cnnForcast' + str(opt) train.saveModel(outFolder, model, nEpoch, modelName=modelName) ypLst = list() df = hydroDL.data.dbCsv.DataframeCsv( rootDB=rootDB, subset='CONUSv4f1', tRange=ty2) yT = df.getData(varT='SMAP_AM', doNorm=False, rmNan=False).squeeze() if 'testLstm' in doLst: df = hydroDL.data.dbCsv.DataframeCsv( rootDB=rootDB, subset='CONUSv4f1', tRange=ty2) x = df.getData( varT=dbCsv.varForcing, varC=dbCsv.varConst, doNorm=True, rmNan=True) model = train.loadModel(outFolder, nEpoch, modelName='lstmForcast') yP = train.testModel(model, x).squeeze() ypLst.append( dbCsv.transNorm(yP, rootDB=rootDB, fieldName='SMAP_AM', fromRaw=False)) if 'testCnn' in doLst: df = hydroDL.data.dbCsv.DataframeCsv( rootDB=rootDB, subset='CONUSv4f1', tRange=tyc2) x = df.getData( varT=dbCsv.varForcing, varC=dbCsv.varConst, doNorm=True, rmNan=True) y = df.getData(varT='SMAP_AM', doNorm=True, rmNan=False) yc = np.copy(y) yc[:, :, 0] = utils.interpNan(yc[:, :, 0], mode='pre') for opt in range(1, 3): modelName = 'cnnForcast' + str(opt)
y1 = df1.getDataObs(doNorm=True, rmNan=False) c1 = df1.getDataConst(varLst=camels.attrLstSel, doNorm=True, rmNan=True) yt1 = df1.getDataObs(doNorm=False, rmNan=False).squeeze() dfz1 = camels.DataframeCsv(subset='all', tRange=[20141231, 20091231]) z1 = dfz1.getDataObs(doNorm=True, rmNan=False) dfz2 = camels.DataframeCsv(subset='all', tRange=[20141227, 20091227]) z2 = dfz2.getDataObs(doNorm=True, rmNan=False) df2 = camels.DataframeCsv(subset='all', tRange=[20100101, 20150101]) x2 = df2.getDataTS(varLst=camels.forcingLst, doNorm=True, rmNan=True) c2 = df2.getDataConst(varLst=camels.attrLstSel, doNorm=True, rmNan=True) yt2 = df2.getDataObs(doNorm=False, rmNan=False).squeeze() model = train.loadModel(outFolder, 100, modelName='test') yp1 = train.testModel(model, x1, c1) yp1 = camels.transNorm(yp1, 'usgsFlow', toNorm=False).squeeze() yp2 = train.testModel(model, x2, c2) yp2 = camels.transNorm(yp2, 'usgsFlow', toNorm=False).squeeze() statErr1 = stat.statError(yp1, yt2) statErr2 = stat.statError(yp2, yt2) dataMap = [statErr2['Corr'], statErr1['Corr'] - statErr2['Corr']] dataTs = [yt2, yp2] t = df2.getT() crd = df2.getGeo() mapNameLst = ['Test Corr', 'Train Corr - Test Corr'] tsNameLst = ['USGS', 'LSTM'] colorMap = None colorTs = None