print('batch: ' + str(k))
xT = torch.from_numpy(
np.concatenate([
xNorm[:, iS[k]:iE[k], :],
np.tile(cNorm[iS[k]:iE[k], :], [nt, 1, 1])
],
axis=-1)).float()
if torch.cuda.is_available():
xT = xT.cuda()
model = model.cuda()
yT = model(xT)[-1, :, :]
yOutLst.append(yT.detach().cpu().numpy())
temp = np.concatenate(yOutLst, axis=0)
yOut = np.ndarray(temp.shape)
for k in range(ny):
yOut[:, k] = transform.transOut(temp[:, k], mtdLstY[k],
statLstY[k])
# save output
dfOut = info
dfOut['train'] = np.nan
dfOut['train'][indTrain] = 1
dfOut['train'][indTest] = 0
varC = dictData['varC']
targetFile = os.path.join(modelFolder, 'target.csv')
# if not os.path.exists(targetFile):
targetDf = pd.merge(dfOut,
pd.DataFrame(data=y, columns=varC),
left_index=True,
right_index=True)
targetDf.to_csv(targetFile)
outFile = os.path.join(modelFolder, 'output_Ep' + str(iEp) + '.csv')
def loadSeq(siteNo, varY, model, optX='F', optT='Y8090', order=(5, 0, 5)):
if model == 'ARMA':
dirAR = os.path.join(kPath.dirWQ, 'modelStat', 'ARMA')
strOrder = '-'.join([str(k) for k in order])
saveFolderName = '{}-{}-{}-{}'.format(optX, optT, varY, strOrder)
saveFolder = os.path.join(dirAR, saveFolderName)
elif model == 'LR':
dirLR = os.path.join(kPath.dirWQ, 'modelStat', 'LR')
saveFolderName = '{}-{}-{}'.format(optX, optT, varY)
saveFolder = os.path.join(dirLR, saveFolderName)
else:
raise Exception('model {} invalid!'.format(model))
predFile = os.path.join(saveFolder, siteNo)
if not os.path.exists(saveFolder):
os.mkdir(saveFolder)
if os.path.exists(predFile):
dfP = pd.read_csv(predFile, index_col=None)
dfP = utils.time.datePdf(dfP)
else:
if optX == 'F':
varX = gridMET.varLst
elif optX == 'QF':
varX = ['00060'] + gridMET.varLst
else:
raise Exception('optX {} invalid!'.format(optX))
dfX = waterQuality.readSiteX(siteNo, varX)
dfY = waterQuality.readSiteY(siteNo, [varY])
# normalize
mtdX = waterQuality.extractVarMtd(varX)
normX, statX = transform.transInAll(dfX.values, mtdX)
dfXN = pd.DataFrame(data=normX, index=dfX.index, columns=dfX.columns)
mtdY = waterQuality.extractVarMtd([varY])
normY, statY = transform.transInAll(dfY.values, mtdY)
dfYN = pd.DataFrame(data=normY, index=dfY.index, columns=dfY.columns)
if optT == 'Y8090':
dfXT = dfXN[dfXN.index < np.datetime64('2000-01-01')]
dfYT = dfYN[dfYN.index < np.datetime64('2000-01-01')]
elif optT == 'Y0010':
dfXT = dfXN[dfXN.index >= np.datetime64('2000-01-01')]
dfYT = dfYN[dfYN.index >= np.datetime64('2000-01-01')]
else:
raise Exception('optT {} invalid!'.format(optT))
# train and test
if model == 'ARMA':
dfPN, resT = trainARMA(dfXT, dfYT, dfXN, dfYN, order)
if model == 'LR':
dfPN = trainLR(dfXT, dfYT, dfXN, dfYN)
yP = transform.transOut(dfPN.values, mtdY[0], statY[0])
dfP = pd.DataFrame(data=yP, index=dfYN.index, columns=dfYN.columns)
# save result, model, stat
dfP.reset_index().to_csv(predFile, index=False)
statFile = os.path.join(saveFolder, siteNo + '_stat.json')
with open(statFile, 'w') as fp:
json.dump(dict(statX=statX, statY=statY), fp, indent=4)
# save model
# if model == 'ARMA':
# modelFile = os.path.join(saveFolder, siteNo+'_model.p')
# resT.save(modelFile)
return dfP
def transOutYC(self, ycP):
if ycP.shape[-1] != 0:
return transform.transOut(ycP, self.statYC)
else:
return None
def transOutY(self, yP):
if yP.shape[-1] != 0:
return transform.transOut(yP, self.statY)
else:
return None
yc = transform.transInAll(ycA, mtdYC, statLst=statYC)
(x, xc) = trainTS.dealNaN((x, xc), [1, 1])
nt = x.shape[0]
xT = torch.from_numpy(np.concatenate([x, np.tile(xc, [nt, 1, 1])],
axis=-1)).float()
if torch.cuda.is_available():
xT = xT.cuda()
model = model.train(mode=False)
# yP, gate = model(xT)
yP, b, gate = model(xT)
yO = yP.detach().cpu().numpy()
# gate = gate.detach().cpu().numpy()
predY = transform.transOut(yO[:, :, 0], mtdY[0], statY[0])
predYC = transform.transOutAll(yO[:, :, 1:], mtdYC, statYC)
obsY = dfY.values
obsYC = dfYC.values
t = dfY.index.values
fig, axes = plt.subplots(4, 1)
axplot.plotTS(axes[0], t, [predY, obsY], styLst='---', cLst='rb')
axes[0].set_title('streamflow')
axes[0].set_xticks([])
codePdf = usgs.codePdf
for k, code in enumerate(codeLst):
axplot.plotTS(axes[k + 1],
t, [predYC[:, 0, k], obsYC[:, k]],
styLst='-*',
cLst='rb')
],
axis=-1)).float()
if torch.cuda.is_available():
xT = xT.cuda()
model = model.cuda()
# yT = model(xT)[:, :, 0]
ycT = model(xT)[-1, :, 1:]
# yPLst.append(yT.detach().cpu().numpy())
ycPLst.append(ycT.detach().cpu().numpy())
# yP = np.concatenate(yPLst, axis=1)
ycP = np.concatenate(ycPLst, axis=0)
# yOut = np.ndarray(yP.shape)
ycOut = np.ndarray(ycP.shape)
# yOut = transform.transOut(yP, mtdLstY[0], statLstY[0])
for k in range(nyc):
ycOut[:, k] = transform.transOut(ycP[:, k], mtdLstYC[k],
statLstYC[k])
# save output
print(2)
dfOut = info
dfOut['train'] = np.nan
dfOut['train'][indTrain] = 1
dfOut['train'][indTest] = 0
varC = dictData['varC']
targetFile = os.path.join(modelFolder, 'target.csv')
# if not os.path.exists(targetFile):
targetDf = pd.merge(dfOut,
pd.DataFrame(data=yc, columns=varC),
left_index=True,
right_index=True)
targetDf.to_csv(targetFile)
print(type(DF) is hydroDL.data.dbBasin.DataModelBasin)
tempFolder = os.path.join(kPath.dirCode, 'temp')
mtdX = dbBasin.io.extractVarMtd(d1.v)
x = d1.X[:, :, 9]
dataIn = np.repeat(x[:, :, None], 6, axis=2)
mtdLst = ['norm', 'log-norm', 'stan', 'log-stan', 'QT', 'log-QT']
q = d1.X[:, :, -1]
# transIn
a = dataIn.copy()
b, dictTran = transform.transIn(a, mtdLst=mtdLst)
d, dictTran = transform.transIn(a, **dictTran)
c = transform.transOut(b, dictTran)
# check hist
nd = len(mtdLst)
bins = 50
indS = 8
fig, axes = plt.subplots(nd, 2, figsize=(4, 8))
for k, var in enumerate(mtdLst):
_ = axes[k, 0].hist(a[:, :, k].flatten(), bins=bins)
_ = axes[k, 1].hist(b[:, :, k].flatten(), bins=bins)
fig.show()
fig, axes = plt.subplots(nd, 2, figsize=(4, 8))
for k, var in enumerate(mtdLst):
_ = axes[k, 0].hist(a[:, indS, k].flatten(), bins=bins)
_ = axes[k, 1].hist(b[:, indS, k].flatten(), bins=bins)