def trainModelTS(outName): outFolder = nameFolder(outName) dictP = loadMaster(outName) # load data wqData = waterQuality.DataModelWQ(dictP['dataName']) varTup = (dictP['varX'], dictP['varXC'], dictP['varY'], dictP['varYC']) dataTup, statTup = wqData.transIn(subset=dictP['trainName'], varTup=varTup) dataTup = trainTS.dealNaN(dataTup, dictP['optNaN']) wrapStat(outName, statTup) # train model [nx, nxc, ny, nyc, nt, ns] = trainTS.getSize(dataTup) if dictP['modelName'] == 'CudnnLSTM': model = rnn.CudnnLstmModel(nx=nx + nxc, ny=ny + nyc, hiddenSize=dictP['hiddenSize']) lossFun = crit.RmseLoss() if torch.cuda.is_available(): lossFun = lossFun.cuda() model = model.cuda() optim = torch.optim.Adadelta(model.parameters()) lossLst = list() nEp = dictP['nEpoch'] sEp = dictP['saveEpoch'] logFile = os.path.join(outFolder, 'log') if os.path.exists(logFile): os.remove(logFile) for k in range(0, nEp, sEp): model, optim, lossEp = trainTS.trainModel(dataTup, model, lossFun, optim, batchSize=dictP['batchSize'], nEp=sEp, cEp=k, logFile=logFile) # save model saveModel(outName, k + sEp, model, optim=optim) lossLst = lossLst + lossEp lossFile = os.path.join(outFolder, 'loss.csv') pd.DataFrame(lossLst).to_csv(lossFile, index=False, header=False)
# doLst.append('test') # doLst.append('post') if 'train' in doLst: # load data df = hydroDL.data.dbCsv.DataframeCsv(rootDB=rootDB, subset='CONUSv4f1', tRange=ty1) x = df.getDataTs(dbCsv.varForcing, doNorm=True, rmNan=True) c = df.getDataConst(dbCsv.varConst, doNorm=True, rmNan=True) y = df.getDataTs('SMAP_AM', doNorm=True, rmNan=False) nx = x.shape[-1] + c.shape[-1] ny = 1 model = rnn.CudnnLstmModel(nx=nx, ny=ny, hiddenSize=64) lossFun = crit.RmseLoss() model = train.trainModel(model, x, y, c, lossFun, nEpoch=nEpoch, miniBatch=[100, 30]) modelName = 'test-LSTM' train.saveModel(outFolder, model, nEpoch, modelName=modelName) for k in dLst: sd = utils.time.t2dt(ty1[0]) - dt.timedelta(days=k) ed = utils.time.t2dt(ty1[1]) - dt.timedelta(days=k) df2 = hydroDL.data.dbCsv.DataframeCsv(rootDB=rootDB, subset='CONUSv4f1',
def trainModelTS(outName): outFolder = nameFolder(outName) dictP = loadMaster(outName) # load data rmFlag = dictP['rmFlag'] if 'rmFlag' in dictP else False wqData = waterQuality.DataModelWQ(dictP['dataName'], rmFlag) varTup = (dictP['varX'], dictP['varXC'], dictP['varY'], dictP['varYC']) dataTup, statTup = wqData.transIn(subset=dictP['trainName'], varTup=varTup) dataTup = trainTS.dealNaN(dataTup, dictP['optNaN']) wrapStat(outName, statTup) # train model [nx, nxc, ny, nyc, nt, ns] = trainTS.getSize(dataTup) # define loss if dictP['crit'] == 'RmseLoss': lossFun = crit.RmseLoss() elif dictP['crit'] == 'RmseLoss2D': lossFun = crit.RmseLoss2D() elif dictP['crit'] == 'SigmaLoss': lossFun = crit.SigmaLoss() ny = ny * 2 nyc = nyc * 2 else: raise RuntimeError('loss function not specified') # define model if dictP['modelName'] == 'CudnnLSTM': model = rnn.CudnnLstmModel(nx=nx + nxc, ny=ny + nyc, hiddenSize=dictP['hiddenSize']) elif dictP['modelName'] == 'LstmModel': model = rnn.LstmModel(nx=nx + nxc, ny=ny + nyc, hiddenSize=dictP['hiddenSize']) elif dictP['modelName'] == 'AgeLSTM': model = rnn.AgeLSTM2(nx=nx + nxc, ny=ny, nyc=nyc, rho=365, nh=dictP['hiddenSize']) else: raise RuntimeError('Model not specified') if torch.cuda.is_available(): lossFun = lossFun.cuda() model = model.cuda() if dictP['optim'] == 'AdaDelta': optim = torch.optim.Adadelta(model.parameters()) else: raise RuntimeError('optimizor function not specified') lossLst = list() nEp = dictP['nEpoch'] sEp = dictP['saveEpoch'] logFile = os.path.join(outFolder, 'log') if os.path.exists(logFile): os.remove(logFile) for k in range(0, nEp, sEp): model, optim, lossEp = trainTS.trainModel(dataTup, model, lossFun, optim, batchSize=dictP['batchSize'], nEp=sEp, cEp=k, logFile=logFile) # save model saveModel(outName, k + sEp, model, optim=optim) lossLst = lossLst + lossEp lossFile = os.path.join(outFolder, 'loss.csv') pd.DataFrame(lossLst).to_csv(lossFile, index=False, header=False)
xLst.append(tempX.values) x = np.stack(xLst, axis=-1).swapaxes(1, 2).astype(np.float32) y = np.stack(yLst, axis=-1).swapaxes(1, 2).astype(np.float32) x[np.where(np.isnan(x))] = -1 ind1 = dfC.index.year <= 2016 xx = x[:, ind1, :] yy = y[:, ind1, :] # training nbatch = 20 nEp = 100 ns = xx.shape[1] nx = xx.shape[-1] ny = yy.shape[-1] model = rnn.LstmModel(nx=nx, ny=ny, hiddenSize=256).cuda() optim = torch.optim.Adadelta(model.parameters()) lossFun = crit.RmseLoss().cuda() nIterEp = int(np.ceil(np.log(0.01) / np.log(1 - nbatch / ns))) lossLst = list() for iEp in range(1, nEp + 1): lossEp = 0 t0 = time.time() for iIter in range(nIterEp): iR = np.random.randint(0, ns, nbatch) xTemp = xx[:, iR, :] yTemp = yy[:, iR, :] xT = torch.from_numpy(xTemp).float().cuda() yT = torch.from_numpy(yTemp).float().cuda() if iEp == 1 and iIter == 0: try: yP = model(xT) except:
def train(mDict): if mDict is str: mDict = readMasterFile(mDict) out = mDict["out"] optData = mDict["data"] optModel = mDict["model"] optLoss = mDict["loss"] optTrain = mDict["train"] # fix the random seed if optTrain["seed"] is None: # generate random seed randomseed = int(np.random.uniform(low=0, high=1e6)) optTrain["seed"] = randomseed print("random seed updated!") else: randomseed = optTrain["seed"] random.seed(randomseed) torch.manual_seed(randomseed) np.random.seed(randomseed) torch.cuda.manual_seed(randomseed) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False # data df, x, y, c = loadData(optData) # x: ngage*nday*nvar # y: ngage*nday*nvar # c: ngage*nvar # temporal test, fill obs nan using LSTM forecast # temp = x[:,:,-1, None] # y[np.isnan(y)] = temp[np.isnan(y)] if c is None: if type(x) is tuple: nx = x[0].shape[-1] else: nx = x.shape[-1] else: if type(x) is tuple: nx = x[0].shape[-1] + c.shape[-1] else: nx = x.shape[-1] + c.shape[-1] ny = y.shape[-1] # loss if eval(optLoss["name"]) is hydroDL.model.crit.SigmaLoss: lossFun = crit.SigmaLoss(prior=optLoss["prior"]) optModel["ny"] = ny * 2 elif eval(optLoss["name"]) is hydroDL.model.crit.RmseLoss: lossFun = crit.RmseLoss() optModel["ny"] = ny elif eval(optLoss["name"]) is hydroDL.model.crit.NSELoss: lossFun = crit.NSELoss() optModel["ny"] = ny elif eval(optLoss["name"]) is hydroDL.model.crit.NSELosstest: lossFun = crit.NSELosstest() optModel["ny"] = ny elif eval(optLoss["name"]) is hydroDL.model.crit.MSELoss: lossFun = crit.MSELoss() optModel["ny"] = ny elif eval(optLoss["name"]) is hydroDL.model.crit.RmseLossCNN: lossFun = crit.RmseLossCNN() optModel["ny"] = ny elif eval(optLoss["name"]) is hydroDL.model.crit.ModifyTrend1: lossFun = crit.ModifyTrend1() optModel["ny"] = ny # model if optModel["nx"] != nx: print("updated nx by input data") optModel["nx"] = nx if eval(optModel["name"]) is hydroDL.model.rnn.CudnnLstmModel: if type(x) is tuple: x = np.concatenate([x[0], x[1]], axis=2) if c is None: nx = x.shape[-1] else: nx = x.shape[-1] + c.shape[-1] optModel["nx"] = nx print("Concatenate input and obs, update nx by obs") model = rnn.CudnnLstmModel( nx=optModel["nx"], ny=optModel["ny"], hiddenSize=optModel["hiddenSize"] ) elif eval(optModel["name"]) is hydroDL.model.rnn.CpuLstmModel: model = rnn.CpuLstmModel( nx=optModel["nx"], ny=optModel["ny"], hiddenSize=optModel["hiddenSize"] ) elif eval(optModel["name"]) is hydroDL.model.rnn.LstmCloseModel: model = rnn.LstmCloseModel( nx=optModel["nx"], ny=optModel["ny"], hiddenSize=optModel["hiddenSize"], fillObs=True, ) elif eval(optModel["name"]) is hydroDL.model.rnn.AnnModel: model = rnn.AnnCloseModel( nx=optModel["nx"], ny=optModel["ny"], hiddenSize=optModel["hiddenSize"] ) elif eval(optModel["name"]) is hydroDL.model.rnn.AnnCloseModel: model = rnn.AnnCloseModel( nx=optModel["nx"], ny=optModel["ny"], hiddenSize=optModel["hiddenSize"], fillObs=True, ) elif eval(optModel["name"]) is hydroDL.model.cnn.LstmCnn1d: convpara = optModel["convNKSP"] model = hydroDL.model.cnn.LstmCnn1d( nx=optModel["nx"], ny=optModel["ny"], rho=optModel["rho"], nkernel=convpara[0], kernelSize=convpara[1], stride=convpara[2], padding=convpara[3], ) elif eval(optModel["name"]) is hydroDL.model.rnn.CNN1dLSTMmodel: daobsOption = optData["daObs"] if type(daobsOption) is list: if len(daobsOption) - 3 >= 7: # using 1dcnn only when number of obs larger than 7 optModel["nobs"] = len(daobsOption) convpara = optModel["convNKS"] model = rnn.CNN1dLSTMmodel( nx=optModel["nx"], ny=optModel["ny"], nobs=optModel["nobs"] - 3, hiddenSize=optModel["hiddenSize"], nkernel=convpara[0], kernelSize=convpara[1], stride=convpara[2], poolOpt=optModel["poolOpt"], ) print("CNN1d Kernel is used!") else: if type(x) is tuple: x = np.concatenate([x[0], x[1]], axis=2) nx = x.shape[-1] + c.shape[-1] optModel["nx"] = nx print("Concatenate input and obs, update nx by obs") model = rnn.CudnnLstmModel( nx=optModel["nx"], ny=optModel["ny"], hiddenSize=optModel["hiddenSize"], ) optModel["name"] = "hydroDL.model.rnn.CudnnLstmModel" print("Too few obserservations, not using cnn kernel") else: raise Exception("CNN kernel used but daobs option is not obs list") elif eval(optModel["name"]) is hydroDL.model.rnn.CNN1dLSTMInmodel: # daobsOption = optData['daObs'] daobsOption = list(range(24)) if type(daobsOption) is list: if len(daobsOption) - 3 >= 7: # using 1dcnn only when number of obs larger than 7 optModel["nobs"] = len(daobsOption) convpara = optModel["convNKS"] model = rnn.CNN1dLSTMInmodel( nx=optModel["nx"], ny=optModel["ny"], # nobs=optModel['nobs']-3, nobs=24, # temporary test hiddenSize=optModel["hiddenSize"], nkernel=convpara[0], kernelSize=convpara[1], stride=convpara[2], poolOpt=optModel["poolOpt"], ) print("CNN1d Kernel is used!") else: if type(x) is tuple: x = np.concatenate([x[0], x[1]], axis=2) nx = x.shape[-1] + c.shape[-1] optModel["nx"] = nx print("Concatenate input and obs, update nx by obs") model = rnn.CudnnLstmModel( nx=optModel["nx"], ny=optModel["ny"], hiddenSize=optModel["hiddenSize"], ) optModel["name"] = "hydroDL.model.rnn.CudnnLstmModel" print("Too few obserservations, not using cnn kernel") else: raise Exception("CNN kernel used but daobs option is not obs list") elif eval(optModel["name"]) is hydroDL.model.rnn.CNN1dLCmodel: # LCrange = optData['lckernel'] # tLCLst = utils.time.tRange2Array(LCrange) if len(x[1].shape) == 2: # for LC-FDC optModel["nobs"] = x[1].shape[-1] elif len(x[1].shape) == 3: # for LC-SMAP--get time step optModel["nobs"] = x[1].shape[1] convpara = optModel["convNKS"] model = rnn.CNN1dLCmodel( nx=optModel["nx"], ny=optModel["ny"], nobs=optModel["nobs"], hiddenSize=optModel["hiddenSize"], nkernel=convpara[0], kernelSize=convpara[1], stride=convpara[2], poolOpt=optModel["poolOpt"], ) print("CNN1d Local calibartion Kernel is used!") elif eval(optModel["name"]) is hydroDL.model.rnn.CNN1dLCInmodel: LCrange = optData["lckernel"] tLCLst = utils.time.tRange2Array(LCrange) optModel["nobs"] = x[1].shape[-1] convpara = optModel["convNKS"] model = rnn.CNN1dLCInmodel( nx=optModel["nx"], ny=optModel["ny"], nobs=optModel["nobs"], hiddenSize=optModel["hiddenSize"], nkernel=convpara[0], kernelSize=convpara[1], stride=convpara[2], poolOpt=optModel["poolOpt"], ) print("CNN1d Local calibartion Kernel is used!") elif eval(optModel["name"]) is hydroDL.model.rnn.CudnnInvLstmModel: # optModel['ninv'] = x[1].shape[-1] optModel["ninv"] = x[1].shape[-1] + c.shape[-1] # Test the inv using attributes model = rnn.CudnnInvLstmModel( nx=optModel["nx"], ny=optModel["ny"], hiddenSize=optModel["hiddenSize"], ninv=optModel["ninv"], nfea=optModel["nfea"], hiddeninv=optModel["hiddeninv"], ) print("LSTMInv model is used!") # train if optTrain["saveEpoch"] > optTrain["nEpoch"]: optTrain["saveEpoch"] = optTrain["nEpoch"] # train model writeMasterFile(mDict) model = trainModel( model, x, y, c, lossFun, nEpoch=optTrain["nEpoch"], miniBatch=optTrain["miniBatch"], saveEpoch=optTrain["saveEpoch"], saveFolder=out, )
def forward(self, x): x = F.relu(self.hidden1(x)) x = F.relu(self.hidden2(x)) x = self.predict(x) return x net = Net(n_feature=x.shape[2], n_hidden=Hid_lay, n_output=1) # define the network net.to(device, dtype=torch.float32) print(net) # define the training model optimizer = torch.optim.Adadelta(net.parameters()) loss_func = crit.RmseLoss( get_length=True) # this is for regression mean squared loss logger = {} logger['rmse_train'] = [] logger['rmse_test'] = [] logger['train_loss'] = [] # define testing def test(epoch): testmse = 0.0 Nsample = 0 for step, (batch_x, batch_y) in enumerate(test_loader): batch_x, batch_y = batch_x.to(device), batch_y.to(device) out = net(batch_x) loss, Nday = loss_func(out, batch_y) tempmse = (loss.item()**2) * Nday