# master.runTrain(masterDict, cudaID=cid % 3, screen='test-DA' + str(nDay)) cid = cid + 1 # test original model caseLst = ['All-90-95'] nDayLst = [3, 10, 30, 100, 365] for nDay in nDayLst: caseLst.append('All-90-95-DA' + str(nDay)) outLst = [os.path.join(pathCamels['Out'], save_path, x) for x in caseLst] subset = 'All' tRange = [19950101, 20000101] predLst = list() for out in outLst: df, pred, obs = master.test(out, tRange=tRange, subset=subset, basinnorm=True, epoch=200) # pred=np.maximum(pred,0) predLst.append(pred) # plot box statDictLst = [stat.statError(x.squeeze(), obs.squeeze()) for x in predLst] # keyLst = list(statDictLst[0].keys()) keyLst = ['Bias', 'RMSE', 'NSE'] dataBox = list() for iS in range(len(keyLst)): statStr = keyLst[iS] temp = list() for k in range(len(statDictLst)): data = statDictLst[k][statStr]
from hydroDL import pathSMAP, master import os from hydroDL.master import default cDir = os.path.dirname(os.path.abspath(__file__)) cDir = r'/home/kxf227/work/GitHUB/pyRnnSMAP/example/' # define training options optData = default.update( default.optDataSMAP, rootDB=os.path.join(cDir, 'data'), subset='CONUSv4f1', tRange=[20150401, 20160401], ) optModel = default.optLstm optLoss = default.optLossSigma optTrain = default.update(master.default.optTrainSMAP, nEpoch=5, saveEpoch=5) out = os.path.join(cDir, 'output', 'CONUSv4f1_sigma') masterDict = master.wrapMaster(out, optData, optModel, optLoss, optTrain) # train master.train(masterDict) # test pred = master.test(out, tRange=[20160401, 20170401], subset='CONUSv4f1')
import os from hydroDL.data import dbCsv from hydroDL.post import plot, stat from hydroDL import master cDir = os.path.dirname(os.path.abspath(__file__)) out = os.path.join(cDir, 'output', 'CONUSv4f1') rootDB = os.path.join(cDir, 'data') nEpoch = 100 tRange = [20160401, 20170401] # load data df, yp, yt = master.test( out, tRange=[20160401, 20170401], subset='CONUSv4f1', epoch=100, reTest=True) yp = yp.squeeze() yt = yt.squeeze() # calculate stat statErr = stat.statError(yp, yt) dataGrid = [statErr['RMSE'], statErr['Corr']] dataTs = [yp, yt] t = df.getT() crd = df.getGeo() mapNameLst = ['RMSE', 'Correlation'] tsNameLst = ['LSTM', 'SMAP'] # plot map and time series plot.plotTsMap( dataGrid, dataTs,
# k = [7, 8, 13] # %% load data and stat kcLst = [7, 8, 13] tRange = [20160401, 20180401] statLst = list() statRefLst = list() for kc in kcLst: tempLst = list() for k in range(1, 18): testName = subsetLst[kc - 1] if k != kc: outName = 'ecoRegion{:02d}{:02d}_v2f1_Forcing'.format(kc, k) else: outName = 'ecoRegion{:02d}_v2f1_Forcing'.format(kc) out = os.path.join(pathSMAP['Out_L3_NA'], 'ecoRegion', outName) df, yp, yt = master.test(out, tRange=tRange, subset=testName) temp = stat.statError(yp[:, :, 0], yt[:, :, 0]) tempLst.append(temp) if k == kc: statRefLst.append(temp) statLst.append(tempLst) # %% plot box keyLst = stat.keyLst ecoLst = ['{:02d}'.format(x) for x in range(1, 18)] caseLst = ['{:02d}'.format(x) for x in [7, 8, 13]] for k in range(len(caseLst)): dataBox = list() key = 'RMSE' for ii in range(len(ecoLst)):
from hydroDL import pathSMAP, master, utils from hydroDL.master import default from hydroDL.post import plot, stat import os import matplotlib.pyplot as plt import numpy as np import torch # train optData = default.update(default.optDataSMAP, rootDB=pathSMAP['DB_L3_NA'], subset='CONUSv4f1', tRange=[20150402, 20180401], daObs=1) optModel = default.optLstmClose optLoss = default.optLossRMSE optTrain = default.update(default.optTrainSMAP, nEpoch=500) out = os.path.join(pathSMAP['Out_L3_NA'], 'DA', 'CONUS_DA_3yr') masterDict = master.wrapMaster(out, optData, optModel, optLoss, optTrain) master.runTrain(masterDict, cudaID=2, screen='DA') # test subset = 'CONUS' tRange = [20150402, 20180401] # out = os.path.join(pathSMAP['Out_L3_NA'], 'DA', 'CONUSv2f1_DA2015') out = os.path.join(pathSMAP['Out_L3_NA'], 'DA', 'CONUS_DA_3yr') df, yf, obs = master.test(out, tRange=tRange, subset=subset, batchSize=100) # /home/kxf227/work/GitHUB/hydroDL-dev/app/closeLoop/fullCONUS.py
dataPred[:, :, k] = pd.read_csv(filePath, dtype=np.float, header=None).values # transform back to the original observation temppred = camels.transNormbyDic(dataPred, "runoff", statDict, toNorm=False) pred = camels.basinNorm(temppred, subset, toNorm=False) elif interfaceOpt == 0: # only for models trained by the pro interface df, pred, obs = master.test( out, tRange=tRange, subset=subset, batchSize=testBatch, basinnorm=True, epoch=TestEPOCH, reTest=True, ) # change the units ft3/s to m3/s obs = obs * 0.0283168 pred = pred * 0.0283168 predLst.append(pred) # the prediction list for all the models # calculate statistic metrics statDictLst = [stat.statError(x.squeeze(), obs.squeeze()) for x in predLst] # Show boxplots of the results plt.rcParams["font.size"] = 14
temppred = camels.transNormbyDic(dataPred, "runoff", statDict, toNorm=False) pred = camels.basinNorm(temppred, np.array(testBasin), toNorm=False) elif interfaceOpt == 0: if migOptLst[icount] is True: # for FDC migration case df, pred, obs = master.test( out, tRange=tRange, subset=testBasin, basinnorm=True, epoch=testEpoch, reTest=True, FDCgage=FDCList[imig], ) imig = imig + 1 else: # for other ordinary cases df, pred, obs = master.test( out, tRange=tRange, subset=testBasin, basinnorm=True, epoch=testEpoch, reTest=True, )
outLst = [os.path.join(rootOut, save_path, x) for x in caseLst] subset = 'All' # 'All': use all the CAMELS gages to test; Or pass the gage list tRange = [20141001, 20161001] # Testing period predLst = list() obsLst = list() predLst_res = list() obsLst_res = list() statDictLst = [] for i, out in enumerate(outLst): #df, pred, obs = master.test(out, TempTarget, forcing_path[i], attr_path[i], tRange=tRange, subset=subset, basinnorm=True, epoch=TestEPOCH, reTest=True) df, pred, obs, x = master.test(out, TempTarget, forcing_path, attr_path, tRange=tRange, subset=subset, basinnorm=False, epoch=TestEPOCH, reTest=True) # change the units ft3/s to m3/s #obs = obs * 0.0283168 #pred = pred * 0.0283168 predLst.append(pred) # the prediction list for all the models obsLst.append(obs) np.save(os.path.join(out, 'pred.npy'), pred) np.save(os.path.join(out, 'obs.npy'), obs) f = np.load( os.path.join(out, 'x.npy') ) # it has been saved previously in the out directory (forcings)
from hydroDL import pathSMAP, master import os optData = master.updateOpt( master.default.optDataCsv, path=pathSMAP['DB_L3_NA'], subset='CONUSv4f1', dateRange=[20150401, 20160331]) optModel = master.default.optLstm optLoss = master.default.optLoss optTrain = master.default.optTrainSMAP out = os.path.join(pathSMAP['Out_L3_Global'], 'regTest') masterDict = master.wrapMaster(out, optData, optModel, optLoss, optTrain) # master.train(masterDict, overwrite=True) pred = master.test( out, tRange=[20160401, 20170331], subset='CONUSv4f1', epoch=400)
optTrain = master.default.optTrainSMAP out = os.path.join(pathSMAP['Out_L3_NA'], 'ecoRegion', subsetLst[k] + '_' + case) masterDict = master.wrapMaster(out, optData, optModel, optLoss, optTrain) master.runTrain(masterDict, cudaID=cid % 3, screen=subsetLst[k]) cid = cid + 1 # master.train(masterDict) # test ss = '' for k in range(len(subsetLst)): for case in caseLst: out = os.path.join(pathSMAP['Out_L3_NA'], 'ecoRegion', subsetLst[k] + '_' + case) try: master.test(out, tRange=[20160401, 20180401], subset=subsetLst[k]) except: ss = ss + 'ecoRegion ' + str(k) + ' case ' + case + '; ' # retrain some models # rtEcoLst = [1, 2, 5, 7, 8, 10, 13, 16, 17] # rtCaseLst = [ # 'Forcing', 'Soilm', 'Soilm', 'Forcing', 'Soilm', 'Forcing', 'Forcing', # 'Forcing', 'Soilm' # ] # rtEcoLst = [1, 4, 7, 16] # rtCaseLst = ['Soilm', 'Soilm', 'Soilm', 'Soilm'] # cid = 2 # for kk in range(len(rtEcoLst)): # k = rtEcoLst[kk] - 1 # case = rtCaseLst[kk]
doLst.append('test') doLst.append('post') saveDir = os.path.join(pathSMAP['dirResult'], 'DA', 'paper') # test # torch.cuda.set_device(2) subset = 'CONUSv2f1' tRange = [20160501, 20161001] outLst = [ 'CONUSv2f1_DA2015', 'CONUSv2f1_DA_2015RK', 'CONUSv2f1_DA_Prcp_2015', 'CONUSv2f1_DA_Prcp_2015RK' ] yfLst = list() for outName in outLst: out = os.path.join(pathSMAP['Out_L3_NA'], 'DA', outName) df, yf, obs = master.test(out, tRange=tRange, subset=subset) yfLst.append(yf.squeeze()) obs = obs.squeeze() # figure out how many days observation lead # maskObs = 1 * ~np.isnan(obs.squeeze()) * (qualSuc.squeeze() == 0) maskObs = 1 * ~np.isnan(obs.squeeze()) maskDay = np.zeros(maskObs.shape).astype(int) ngrid, nt = maskObs.shape for j in range(ngrid): temp = 0 for i in range(nt): maskDay[j, i] = temp if maskObs[j, i] == 1: temp = 1 else:
caseLst2 = ['Forcing', 'Soilm'] yrLst = [20160401, 20170401] statLst = list() for k in range(len(subsetLst)): testName = subsetLst[k] tempLst = list() for case1 in caseLst1: for case2 in caseLst2: if case1 == 'Local': outName = outLst[k] + '_' + case2 elif case1 == 'Global': outName = 'Global_v4f1_y1' + '_' + case2 out = os.path.join(pathSMAP['Out_L3_Global'], outName) df, yp, yt = master.test(out, tRange=yrLst, subset=testName, epoch=500) temp = stat.statError(yp[:, :, 0], yt[:, :, 0]) tempLst.append(temp) statLst.append(tempLst) # plot box keyLst = stat.keyLst caseLst = list() for case1 in caseLst1: for case2 in caseLst2: caseLst.append(case1 + ' ' + case2) for k in range(len(keyLst)): dataBox = list() key = keyLst[k]
rootDB=pathSMAP['DB_L3_NA'], target=['SMAP_AM', 'SOILM_0-10_NOAH'], subset='CONUSv4f1', tRange=[20150401, 20160401]) optModel = default.optLstm optLoss = default.optLossSigma optTrain = default.update(default.optTrainSMAP, nEpoch=100) out = os.path.join(cDir, 'output', 'CONUSv4f1_multi') masterDict = wrapMaster(out, optData, optModel, optLoss, optTrain) # train # train(masterDict) # runTrain(masterDict, cudaID=2, screen='LSTM-multi') # test df, yp, yt, sigma = test(out, tRange=[20160401, 20170401], subset='CONUSv4f1') # plot ts MAP dataGrid = list() dataTs = list() for k in range(2): statErr = stat.statError(yp[:, :, k], yt[:, :, k]) dataGrid.append(statErr['RMSE']) dataTs.append([yp[:, :, k], yt[:, :, k]]) t = df.getT() crd = df.getGeo() mapNameLst = ['RMSE ', 'RMSE'] tsNameLst = ['LSTM', 'SMAP'] plot.plotTsMap(dataGrid, dataTs, lat=crd[0],
cDir = os.path.dirname(os.path.abspath(__file__)) cDir = r'/home/kxf227/work/GitHUB/pyRnnSMAP/example/' rootDB = os.path.join(cDir, 'data') nEpoch = 500 out = os.path.join(cDir, 'output', 'CONUSv4f1') tRange = [20160401, 20170401] # load data df = dbCsv.DataframeCsv( rootDB=rootDB, subset='CONUSv4f1', tRange=tRange) yt = df.getData(varT='SMAP_AM', doNorm=False, rmNan=False) yt = yt.squeeze() yp = master.test( out, tRange=[20160401, 20170401], subset='CONUSv4f1', epoch=500) yp = yp.squeeze() # calculate stat statErr = stat.statError(yp, yt) dataGrid = [statErr['RMSE'], statErr['Corr']] dataTs = [yp, yt] t = df.getT() crd = df.getGeo() mapNameLst = ['RMSE', 'Correlation'] tsNameLst = ['LSTM', 'SMAP'] colorMap = None colorTs = None # plot map and time series plot.plotTsMap(
from hydroDL.post import axplot, stat, figplot import os import matplotlib.pyplot as plt import numpy as np import torch import matplotlib matplotlib.rcParams.update({'font.size': 14}) matplotlib.rcParams.update({'lines.linewidth': 2}) matplotlib.rcParams.update({'lines.markersize': 6}) # test subset = 'CONUSv2f1' out = os.path.join(pathSMAP['Out_L3_NA'], 'DA', 'CONUSv2f1_DA2015') df1, yf1, obs1 = master.test(out, tRange=[20150402, 20180401], subset=subset, batchSize=100) df2, yf2, obs2 = master.test(out, tRange=[20160401, 20180401], subset=subset, batchSize=100) out = os.path.join(pathSMAP['Out_L3_NA'], 'DA', 'CONUSv2f1_LSTM2015') df1, yp1, obs1 = master.test(out, tRange=[20150402, 20180401], subset=subset, batchSize=100) df2, yp2, obs2 = master.test(out, tRange=[20160401, 20180401], subset=subset, batchSize=100)
out = os.path.join(pathSMAP['Out_L3_NA'], 'DA', 'CONUSv2f1') masterDict = master.wrapMaster(out, optData, optModel, optLoss, optTrain) master.runTrain(masterDict, cudaID=0, screen='LSTM') # test if 'test' in doLst: torch.cuda.set_device(2) outLst = [ os.path.join(pathSMAP['Out_L3_NA'], 'DA', 'CONUSv2f1_d' + str(nd)) for nd in dLst ] subset = 'CONUSv2f1' tRange = [20160501, 20170501] predLst = list() outLSTM = os.path.join(pathSMAP['Out_L3_NA'], 'DA', 'CONUSv2f1') df, pred, obs = master.test( outLSTM, tRange=tRange, subset=subset, batchSize=100) predLst.append(pred.squeeze()) for out in outLst: df, pred, obs = master.test( out, tRange=tRange, subset=subset, batchSize=100) predLst.append(pred.squeeze()) obs = obs.squeeze() # plot box - latency # if 'post' in doLst: caseLst = ['Predict'] + ['Nowcast ' + str(nd) + 'd latency' for nd in dLst] statLst1 = [stat.statError(x, obs) for x in predLst] keyLst = list(statLst1[0].keys()) dataBox = list() for iS in range(len(keyLst)): statStr = keyLst[iS]