def scm(model, datasetfile, args):
"""
SCM type run with qt_next prediction model
"""
nn_data = data_io.Data_IO_validation(args.region,
args.nlevs,
datasetfile,
args.locations['normaliser_loc'],
xvars=args.xvars,
yvars=args.yvars,
yvars2=args.yvars2,
add_adv=False)
x, y, y2, xmean, xstd, ymean, ystd, ymean2, ystd2 = nn_data.get_data()
# model = set_model(args)
x_split = nn_data.split_data(x, xyz='x')
qnext_ml = x_split['qtot'].data.numpy()
tnext_ml = x_split['theta'].data.numpy()
q_in = x_split['qtot'][0]
t_in = x_split['theta'][0]
qadv, tadv, swtoa, shf, lhf = x_split['qadv'], x_split[
'theta_adv'], x_split['sw_toa'], x_split['shf'], x_split['lhf']
# swtoa, shf, lhf = x_split['sw_toa'], x_split['shf'], x_split['lhf']
for t in range(len(x)):
# prediction
# print("q_in, q_true: ", q_in.data.numpy()[0], x_split['qtot'].data.numpy()[t,0])
qnext_ml[t] = q_in.data.numpy()[:]
# tnext_ml[t] = t_in.data.numpy()[:]
inputs = torch.cat(
[q_in, qadv[t], t_in, tadv[t], swtoa[t], shf[t], lhf[t]])
# inputs = torch.cat([q_in,t_in,swtoa[t],shf[t],lhf[t]])
yp_ = model(inputs)
yp_split = nn_data.split_data(yp_, xyz='y')
q_in = yp_split['qtot_next']
# t_in = yp_split['theta_next']
# print("q_ml:", q_in.data.numpy()[0])
yt_inverse = nn_data._inverse_transform(y, ymean, ystd)
yt_inverse_split = nn_data.split_data(yt_inverse, xyz='y')
qnext_inv = yt_inverse_split['qtot_next']
# tnext_inv = yt_inverse_split['theta_next']
# yp = torch.from_numpy(np.concatenate([qnext_ml, tnext_ml], axis=1))
yp = torch.from_numpy(qnext_ml)
yp_inverse = nn_data._inverse_transform(yp, ymean, ystd)
yp_inverse_split = nn_data.split_data(yp_inverse, xyz='y')
qnext_ml_inv = yp_inverse_split['qtot_next']
# tnext_ml_inv = yp_inverse_split['theta_next']
output = {
'qtot_next': qnext_inv.data.numpy(),
'qtot_next_ml': qnext_ml_inv.data.numpy(),
# 'theta_next':tnext_inv.data.numpy(),
# 'theta_next_ml':tnext_ml_inv.data.numpy()
}
hfilename = args.model_name.replace('.tar', '_scm.hdf5')
with h5py.File(hfilename, 'w') as hfile:
for k, v in output.items():
hfile.create_dataset(k, data=v)
def evaluate_add_adv(model, datasetfile, args):
nn_data = data_io.Data_IO_validation(args.region,
args.nlevs,
datasetfile,
args.locations['normaliser_loc'],
add_adv=True)
x, y, y2, xmean, xstd, ymean, ystd, ymean2, ystd2 = nn_data.get_data()
# model = set_model(args)
yp = model(x)
yinv = nn_data._inverse_transform(y, ymean, ystd)
ypinv = nn_data._inverse_transform(yp, ymean, ystd)
yinv_split = nn_data.split_data(yinv, xyz='y')
ypinv_split = nn_data.split_data(ypinv, xyz='y')
y_split = nn_data.split_data(y, xyz='y')
yp_split = nn_data.split_data(yp, xyz='y')
# ['qtot', 'qadv', 'theta', 'theta_adv', 'sw_toa', 'shf', 'lhf']
# ['qphys', 'theta_phys']
qphys_predict_denorm = ypinv_split['qphys']
qphys_test_denorm = yinv_split['qphys']
theta_phys_predict_denorm = ypinv_split['theta_phys']
theta_phys_test_denorm = yinv_split['theta_phys']
qphys_predict_norm = yp_split['qphys']
qphys_test_norm = y_split['qphys']
theta_phys_predict_norm = yp_split['theta_phys']
theta_phys_test_norm = y_split['theta_phys']
hfilename = args.model_name.replace('.tar', '_qtphys.hdf5')
output = {
'qphys_predict': qphys_predict_denorm.data.numpy(),
'qphys_test': qphys_test_denorm.data.numpy(),
'tphys_predict': theta_phys_predict_denorm.data.numpy(),
'tphys_test': theta_phys_test_denorm.data.numpy(),
'qphys_predict_norm': qphys_predict_norm.data.numpy(),
'qphys_test_norm': qphys_test_norm.data.numpy(),
'tphys_predict_norm': theta_phys_predict_norm.data.numpy(),
'tphys_test_norm': theta_phys_test_norm.data.numpy(),
}
with h5py.File(hfilename, 'w') as hfile:
for k, v in output.items():
hfile.create_dataset(k, data=v)
def evaluate_tnext(model, datasetfile, args):
nn_data = data_io.Data_IO_validation(args.region,
args.nlevs,
datasetfile,
args.locations['normaliser_loc'],
xvars=args.xvars,
yvars=args.yvars,
yvars2=args.yvars2,
add_adv=False)
x, y, y2, xmean, xstd, ymean, ystd, ymean2, ystd2 = nn_data.get_data()
# model = set_model(args)
yp = model(x)
xinv = nn_data._inverse_transform(x, xmean, xstd)
xinv_split = nn_data.split_data(xinv, xyz='x')
yinv = nn_data._inverse_transform(y, ymean, ystd)
ypinv = nn_data._inverse_transform(yp, ymean, ystd)
yinv_split = nn_data.split_data(yinv, xyz='y')
ypinv_split = nn_data.split_data(ypinv, xyz='y')
y_split = nn_data.split_data(y, xyz='y')
yp_split = nn_data.split_data(yp, xyz='y')
y2inv = nn_data._inverse_transform(y2, ymean2, ystd2)
y2inv_split = nn_data.split_data(y2inv, xyz='y2')
# ['qtot', 'qadv', 'theta', 'theta_adv', 'sw_toa', 'shf', 'lhf']
# ['qphys', 'theta_phys']
qtot_denorm = xinv_split['qtot']
theta_denorm = xinv_split['theta']
# qadv_denorm = xinv_split['qadv']
# theta_adv_denorm = xinv_split['theta_adv']
qtotn_predict_denorm = ypinv_split['qtot_next']
qtotn_test_denorm = yinv_split['qtot_next']
thetan_predict_denorm = ypinv_split['theta_next']
thetan_test_denorm = yinv_split['theta_next']
qtotn_predict_norm = yp_split['qtot_next']
qtotn_test_norm = y_split['qtot_next']
thetan_predict_norm = yp_split['theta_next']
thetan_test_norm = y_split['theta_next']
qphys_denorm = y2inv_split['qphys']
theta_phys_denorm = y2inv_split['theta_phys']
# qphys_ml = qtotn_predict_denorm - qtot_denorm - qadv_denorm
# theta_phys_ml = thetan_predict_denorm - theta_denorm - theta_adv_denorm
hfilename = args.model_name.replace('.tar', '_qtphys.hdf5')
output = {
'qtotn_predict': qtotn_predict_denorm.data.numpy(),
'qtotn_test': qtotn_test_denorm.data.numpy(),
'thetan_predict': thetan_predict_denorm.data.numpy(),
'thetan_test': thetan_test_denorm.data.numpy(),
'qtotn_predict_norm': qtotn_predict_norm.data.numpy(),
'qtotn_test_norm': qtotn_test_norm.data.numpy(),
'thetan_predict_norm': thetan_predict_norm.data.numpy(),
'thetan_test_norm': thetan_test_norm.data.numpy(),
# 'qphys_ml':qphys_ml.data.numpy(),
'qphys': qphys_denorm.data.numpy(),
# 'theta_phys_ml':theta_phys_ml.data.numpy(),
'theta_phys': theta_phys_denorm.data.numpy(),
'qtot': qtot_denorm.data.numpy(),
'theta': theta_denorm.data.numpy()
}
with h5py.File(hfilename, 'w') as hfile:
for k, v in output.items():
hfile.create_dataset(k, data=v)
def evaluate_qphys(model, datasetfile, xpca, ypca, args):
nn_data = data_io.Data_IO_validation(args.region,
args.nlevs,
datasetfile,
args.locations['normaliser_loc'],
xvars=args.xvars,
yvars=args.yvars,
yvars2=args.yvars2,
add_adv=False)
x, y, y2, xmean, xstd, ymean, ystd, ymean2, ystd2 = nn_data.get_data()
xpcs, y2pcs = torch.from_numpy(xpca.transform(x)), torch.from_numpy(
ypca.transform(y2))
# model = set_model(args)
# yp = model(x)
yppcs = model(xpcs)
yp = torch.from_numpy(ypca.inverse_transform(yppcs.data.numpy()))
# print(yp[0:100],y2[0:100])
xinv = nn_data._inverse_transform(x, xmean, xstd)
yinv = nn_data._inverse_transform(y2, ymean2, ystd2)
ypinv = nn_data._inverse_transform(yp, ymean2, ystd2)
xinv_split = nn_data.split_data(xinv, xyz='x')
yinv_split = nn_data.split_data(yinv, xyz='y2')
ypinv_split = nn_data.split_data(ypinv, xyz='y2')
x_split = nn_data.split_data(x, xyz='x')
y_split = nn_data.split_data(y2, xyz='y2')
yp_split = nn_data.split_data(yp, xyz='y2')
# ['qtot', 'qadv', 'theta', 'theta_adv', 'sw_toa', 'shf', 'lhf']
# ['qphys', 'theta_phys']
qphys_predict_denorm = ypinv_split['qphys']
qphys_test_denorm = yinv_split['qphys']
# theta_phys_predict_denorm = ypinv_split['theta_phys']
# theta_phys_test_denorm = yinv_split['theta_phys']
qphys_predict_norm = yp_split['qphys']
qphys_test_norm = y_split['qphys']
# theta_phys_predict_norm = yp_split['theta_phys']
# theta_phys_test_norm = y_split['theta_phys']
qtot_test_norm = x_split['qtot']
qadv_test_norm = x_split['qadv']
theta_test_norm = x_split['theta']
theta_adv_test_norm = x_split['theta_adv']
qtot_test_denorm = xinv_split['qtot']
qadv_test_denorm = xinv_split['qadv']
theta_denorm = xinv_split['theta']
theta_adv_denorm = xinv_split['theta_adv']
hfilename = args.model_name.replace('.tar', '_qphys.hdf5')
output = {
'qphys_predict': qphys_predict_denorm.data.numpy(),
'qphys_test': qphys_test_denorm.data.numpy(),
'qphys_predict_norm': qphys_predict_norm.data.numpy(),
'qphys_test_norm': qphys_test_norm.data.numpy(),
'qtot_test_norm': qtot_test_norm.data.numpy(),
'qadv_test_norm': qadv_test_norm.data.numpy(),
'theta_test_norm': theta_test_norm.data.numpy(),
'theta_adv_test_norm': theta_adv_test_norm.data.numpy(),
'qtot_test': qtot_test_denorm.data.numpy(),
'qadv_test': qadv_test_denorm.data.numpy(),
'theta_test': theta_denorm.data.numpy(),
'theta_adv_test': theta_adv_denorm.data.numpy()
}
with h5py.File(hfilename, 'w') as hfile:
for k, v in output.items():
hfile.create_dataset(k, data=v)
100 * (iteration / float(total)))
filledLength = int(length * iteration // total)
bar = fill * filledLength + '-' * (length - filledLength)
print('\r%s |%s| %s%% %s' % (prefix, bar, percent, suffix), end=printEnd)
# Print New Line on Complete
if iteration == total:
print()
# region = "50S69W"
# Data normalizer class
nt = normalize.Normalizers(locations)
# Training and testing data class
# nn_data = data_io.Data_IO(region, locations)
nn_data = data_io.Data_IO_validation(region, locations)
nn_data.get_data(15)
# Define the Model
# n_inputs,n_outputs=140,70
nlevs = 45
in_features, nb_classes = (nlevs * 4 + 3), (nlevs * 2)
nb_hidden_layer = args.nhdn_layers
hidden_size = int(0.66 * in_features + nb_classes)
mlp = model.MLP(in_features, nb_classes, nb_hidden_layer, hidden_size)
# mlp = model.MLP_BN(in_features, nb_classes, nb_hidden_layer, hidden_size)
model_name = args.model_file
# model_name = "q_qadv_t_tadv_swtoa_lhf_shf_qtphys_{0}_lyr_{1}_in_{2}_out_{3}_hdn_{4}_epch_{5}_btch_{6}_mink_vlr.tar".format(str(nb_hidden_layer).zfill(3),
# str(in_features).zfill(3),
# str(nb_classes).zfill(3),
# str(hidden_size).zfill(4),
def scm_diff(qmodel, tmodel, datasetfile, qargs, targs, subdomain):
"""
SCM type run with two models
one model for q prediction and nother for theta perdiction
"""
qnn_data = data_io.Data_IO_validation(
qargs.region,
qargs.nlevs,
datasetfile,
qargs.locations['normaliser_loc'],
xvars=qargs.xvars,
yvars=qargs.yvars,
# yvars2=qargs.yvars2,
add_adv=False,
no_norm=False,
fmin=0,
fmax=100)
tnn_data = data_io.Data_IO_validation(
targs.region,
targs.nlevs,
datasetfile,
targs.locations['normaliser_loc'],
xvars=targs.xvars,
yvars=targs.yvars,
# yvars2=targs.yvars2,
add_adv=False,
no_norm=False,
fmin=0,
fmax=100)
# mplier = targs.xvar_multiplier
# mplier[0] = 4.
# targs.xvar_multiplier = mplier
print("Q model x: ", qargs.xvars)
print("Q model xmul", qargs.xvar_multiplier)
print("Q Model y: ", qargs.yvars)
print("Q Model ymul: ", qargs.yvar_multiplier)
print("T Model x: ", targs.xvars)
print("T Model xmul: ", targs.xvar_multiplier)
print("T Model y: ", targs.yvars)
print("T Model y: ", targs.yvar_multiplier)
qx, qy, qy2, qxmean, qxstd, qymean, qystd, qymean2, qystd2 = qnn_data.get_data(
)
tx, ty, ty2, txmean, txstd, tymean, tystd, tymean2, tystd2 = tnn_data.get_data(
)
# model = set_model(args)
qx_split = qnn_data.split_data(qx, xyz='x')
qyt_split = qnn_data.split_data(qy, xyz='y')
tx_split = tnn_data.split_data(tx, xyz='x')
tyt_split = tnn_data.split_data(ty, xyz='y')
# yt_inverse = nn_data._inverse_transform(y,ymean,ystd)
# qyt_inverse = qy
qyt_inverse = qnn_data._inverse_transform(qy, qymean, qystd)
qyt_inverse_split = qnn_data.split_data(qyt_inverse, xyz='y')
# tyt_inverse = ty
tyt_inverse = tnn_data._inverse_transform(ty, tymean, tystd)
tyt_inverse_split = tnn_data.split_data(tyt_inverse, xyz='y')
# qnext = qyt_split['qtot'][:-1]
# qnext_inv = qyt_inverse_split['qtot'][:-1]
qnext = qyt_split['qv'][:-1]
qnext_inv = qyt_inverse_split['qv'][:-1]
tnext = tyt_split['theta'][:-1]
tnext_inv = tyt_inverse_split['theta'][:-1]
tx_inv_split = tnn_data._inverse_transform_split(tx_split,
txmean,
txstd,
xyz='x')
qx_inv_split = qnn_data._inverse_transform_split(qx_split,
qxmean,
qxstd,
xyz='x')
# qx_inv_split = qnn_data.split_data(qx,xyz='x')
# tx_inv_split = tnn_data.split_data(tx,xyz='x')
# qtot_inv = tx_inv_split['qtot'][:-1]
qtot_inv = tx_inv_split['qv'][:-1]
theta_inv = qx_inv_split['theta'][:-1]
qoutput = []
qoutput.append(qnext[0])
# qoutput.append(qnext[1])
toutput = []
toutput.append(tnext[0])
# toutput.append(tnext[1])
qdifflist = []
for v, m in zip(qargs.xvars, qargs.xvar_multiplier):
qvdiff = (qx_split[v][1:] - qx_split[v][:-1]) * m
qdifflist.append(qvdiff[:-1])
tdifflist = []
for v, m in zip(targs.xvars, targs.xvar_multiplier):
tvdiff = (tx_split[v][1:] - tx_split[v][:-1]) * m
tdifflist.append(tvdiff[:-1])
qxcopy = torch.cat(qdifflist, dim=1)
txcopy = torch.cat(tdifflist, dim=1)
qdiff_output = []
qdiff = tdifflist[0]
qdiff_output.append((qdiff[0]))
tdiff_output = []
tdiff = qdifflist[0]
tdiff_output.append((tdiff[0]))
# plotX(txcopy, qargs)
# sys.exit(0)
for t in range(1, len(qdiff) - 1, 1):
typ_ = tmodel(txcopy[t - 1].reshape(1, -1))
if t > 2:
qxcopy[t - 1, :qargs.nlevs] = typ_
# qxcopy[t-1,:qargs.nlevs] = qdiff_output[t-2]
# qxcopy[t-1,qargs.nlevs:qargs.nlevs*2] = typ_
qyp_ = qmodel(qxcopy[t - 1].reshape(1, -1))
qxcopy[t, :qargs.nlevs] = qyp_
if t > 2:
txcopy[t, :targs.nlevs] = qyp_
if t > 2:
qnew = (qoutput[t - 1] + qyp_[:qargs.nlevs] / torch.Tensor(
qargs.yvar_multiplier)[:]) * 1.0 + (qoutput[t - 2] * 0.0)
tnew = (toutput[t - 1] + typ_[:targs.nlevs] / torch.Tensor(
targs.yvar_multiplier)[:]) * 1.0 + (toutput[t - 2] * 0.0)
else:
qnew = qoutput[t - 1] + qyp_[:qargs.nlevs] / torch.Tensor(
qargs.yvar_multiplier)[:]
tnew = toutput[t - 1] + typ_[:targs.nlevs] / torch.Tensor(
targs.yvar_multiplier)[:]
negative_values = torch.where(qnew < 0.)
if len(negative_values[0]) > 0:
qnew[negative_values] = 1.e-6 #output[t-1][negative_values]
# yp_[negative_values] = diff_output[t-1][negative_values]
# qoutput.append(qnew)
qoutput.append(qnew.reshape(-1))
qdiff_output.append(qyp_)
toutput.append(tnew.reshape(-1))
tdiff_output.append(typ_[:targs.nlevs])
# yp = torch.from_numpy(np.concatenate([qnext_ml, tnext_ml], axis=1))
qyp = torch.stack(qoutput)
typ = torch.stack(toutput)
qnext_ml_inv = qnn_data._inverse_transform(qyp, qymean, qystd)
tnext_ml_inv = tnn_data._inverse_transform(typ, tymean, tystd)
output = {
'qtot_next': qnext_inv.data.numpy(),
'qtot_next_ml': qnext_ml_inv.data.numpy(),
'qtot': qtot_inv.data.numpy(),
'theta': theta_inv.data.numpy(),
'theta_next': tnext_inv.data.numpy(),
'theta_next_ml': tnext_ml_inv.data.numpy()
}
hfilename = qargs.model_name.replace(
'.tar', '_scm_2m_{0}.hdf5'.format(str(subdomain).zfill(3)))
with h5py.File(hfilename, 'w') as hfile:
for k, v in output.items():
hfile.create_dataset(k, data=v)
def scm_diff_multiout(qmodel, tmodel, datasetfile, qargs, targs):
"""
SCM type run with two models
one model for q prediction and nother for theta perdiction
"""
qnn_data = data_io.Data_IO_validation(
qargs.region,
qargs.nlevs,
datasetfile,
qargs.locations['normaliser_loc'],
xvars=qargs.xvars,
yvars=qargs.yvars,
# yvars2=qargs.yvars2,
add_adv=False,
no_norm=False)
tnn_data = data_io.Data_IO_validation(
targs.region,
targs.nlevs,
datasetfile,
targs.locations['normaliser_loc'],
xvars=targs.xvars,
yvars=targs.yvars,
# yvars2=targs.yvars2,
add_adv=False,
no_norm=False)
print("Q model x: ", qargs.xvars)
print("Q model xmul", qargs.xvar_multiplier)
print("Q Model y: ", qargs.yvars)
print("Q Model ymul: ", qargs.yvar_multiplier)
print("T Model x: ", targs.xvars)
print("T Model xmul: ", targs.xvar_multiplier)
print("T Model y: ", targs.yvars)
print("T Model y: ", targs.yvar_multiplier)
qx, qy, qy2, qxmean, qxstd, qymean, qystd, qymean2, qystd2 = qnn_data.get_data(
)
tx, ty, ty2, txmean, txstd, tymean, tystd, tymean2, tystd2 = tnn_data.get_data(
)
# model = set_model(args)
qx_split = qnn_data.split_data(qx, xyz='x')
qyt_split = qnn_data.split_data(qy, xyz='y')
tx_split = tnn_data.split_data(tx, xyz='x')
tyt_split = tnn_data.split_data(ty, xyz='y')
# yt_inverse = nn_data._inverse_transform(y,ymean,ystd)
qyt_inverse = qy
# qyt_inverse = qnn_data._inverse_transform(qy,qymean,qystd)
qyt_inverse_split = qnn_data.split_data(qyt_inverse, xyz='y')
tyt_inverse = ty
# tyt_inverse = tnn_data._inverse_transform(ty,tymean,tystd)
tyt_inverse_split = tnn_data.split_data(tyt_inverse, xyz='y')
qnext = qyt_split['qtot'][:-1]
qnext_inv = qyt_inverse_split['qtot'][:-1]
tnext = tyt_split['theta'][:-1]
tnext_inv = tyt_inverse_split['theta'][:-1]
# x_inv_split = nn_data._inverse_transform_split(x_split,xmean,xstd,xyz='x')
qx_inv_split = qnn_data.split_data(qx, xyz='x')
tx_inv_split = tnn_data.split_data(tx, xyz='x')
qtot_inv = tx_inv_split['qtot'][:-1]
theta_inv = qx_inv_split['theta'][:-1]
qoutput = []
qoutput.append(qnext[0])
# qoutput.append(qnext[1])
toutput = []
toutput.append(tnext[0])
# toutput.append(tnext[1])
qdifflist = []
for v, m in zip(qargs.xvars, qargs.xvar_multiplier):
qvdiff = (qx_split[v][1:] - qx_split[v][:-1]) * m
qdifflist.append(qvdiff[:-1])
tdifflist = []
for v, m in zip(targs.xvars, targs.xvar_multiplier):
tvdiff = (tx_split[v][1:] - tx_split[v][:-1]) * m
tdifflist.append(tvdiff[:-1])
qxcopy = torch.cat(qdifflist, dim=1)
txcopy = torch.cat(tdifflist, dim=1)
qdiff_output = []
qdiff = tdifflist[0]
qdiff_output.append((qdiff[0]))
tdiff_output = []
tdiff = qdifflist[0]
tdiff_output.append((tdiff[0]))
# plotX(x)
# sys.exit(0)
for t in range(1, len(qdiff) - 1, 1):
# qyp_ = qmodel(qxcopy[t-1])
# qyp_1, qyp_2, qyp_3, qyp_4 = qmodel(qxcopy[t-1])
# qyp_ = qyp_1*0.50 + qyp_2*0.50 + qyp_3*0.0 + qyp_4*0.0
typ_1, typ_2, typ_3 = tmodel(txcopy[t - 1])
typ_ = (typ_1 + typ_2 + typ_3) / 3.
# typ_ = typ_2
qxcopy[t - 1, :qargs.nlevs] = typ_ * 10.
qyp_1, qyp_2, qyp_3 = qmodel(qxcopy[t - 1])
qyp_ = (qyp_1 + qyp_2 + qyp_3) / 3.
# qyp_ = qyp_2
txcopy[t, :targs.nlevs] = qyp_ * 10.
# qyp_ = qyp_1*0.33 + qyp_2*0.33 + qyp_3*0.33
# typ_ = tmodel(txcopy[t-1])
# qyp_ = (qyp_ + qdiff_output[t-1])/2.
# typ_ = (typ_ + tdiff_output[t-1])/2.
# qnew = qoutput[t-1] + qyp_[:qargs.nlevs]/torch.Tensor(qargs.yvar_multiplier)[:]
qnew1 = qoutput[t - 1] + qyp_1[:qargs.nlevs] / torch.Tensor(
qargs.yvar_multiplier)[:]
qnew2 = qoutput[t - 1] + qyp_2[:qargs.nlevs] / torch.Tensor(
qargs.yvar_multiplier)[:]
qnew3 = qoutput[t - 1] + qyp_3[:qargs.nlevs] / torch.Tensor(
qargs.yvar_multiplier)[:]
qnew = (qnew1 + qnew2 + qnew3) / 3.
# qnew = (qnew2+qnew3)/2.
# qnew = qnew2
# qnew = (qnew2 + qnew3)/2.
# qyp_ = (qoutput[t-1] - qnew)*10.
# qyp_ = qyp_2
# qyp_ = (qyp_1 + qyp_2 + qyp_3)/3.
# print(t)
# print("Q", qyp_[0], qyp_1[0], qyp_2[0], qyp_3[0], txcopy[t,0])
# qyp_ = (qyp_1 + qyp_2 + qyp_3)/3.
# tnew = toutput[t-1] + typ_[:targs.nlevs]/torch.Tensor(targs.yvar_multiplier)[:]
tnew1 = toutput[t - 1] + typ_1[:targs.nlevs] / torch.Tensor(
targs.yvar_multiplier)[:]
tnew2 = toutput[t - 1] + typ_2[:targs.nlevs] / torch.Tensor(
targs.yvar_multiplier)[:]
tnew3 = toutput[t - 1] + typ_3[:targs.nlevs] / torch.Tensor(
targs.yvar_multiplier)[:]
# tnew = (tnew2 + tnew3)/2.
tnew = (tnew1 + tnew2 + tnew3) / 3.
# tnew = tnew2
# typ_ = (toutput[t-1] - tnew)*10.
# typ_ = (typ_1 + typ_2 + typ_3)/3.
# typ_ = typ_2
# print("T", typ_[0], typ_1[0], typ_2[0], typ_3[0], qxcopy[t,0])
negative_values = torch.where(qnew < 0.)
# print(negative_values[0])
if len(negative_values[0]) > 0:
qnew[negative_values] = 1.e-6 #output[t-1][negative_values]
# yp_[negative_values] = diff_output[t-1][negative_values]
qoutput.append(qnew)
qdiff_output.append(qyp_)
# qxcopy[t,:qargs.nlevs] = typ_*10.
# qxcopy[t,:qargs.nlevs] = qyp_
# qxcopy[t,qargs.nlevs:qargs.nlevs*2] = typ_
toutput.append(tnew)
tdiff_output.append(typ_[:targs.nlevs])
# txcopy[t,:targs.nlevs] = typ_*10.
# yp = torch.from_numpy(np.concatenate([qnext_ml, tnext_ml], axis=1))
qyp = torch.stack(qoutput)
typ = torch.stack(toutput)
qnext_ml_inv = qyp
tnext_ml_inv = typ
output = {
'qtot_next': qnext_inv.data.numpy(),
'qtot_next_ml': qnext_ml_inv.data.numpy(),
'qtot': qtot_inv.data.numpy(),
'theta': theta_inv.data.numpy(),
'theta_next': tnext_inv.data.numpy(),
'theta_next_ml': tnext_ml_inv.data.numpy()
}
hfilename = qargs.model_name.replace('.tar', '_scm_2m.hdf5')
with h5py.File(hfilename, 'w') as hfile:
for k, v in output.items():
hfile.create_dataset(k, data=v)
def scm_diff_qt_diag(model, datasetfile, args):
"""
SCM type run with qt_next prediction model
"""
nn_data = data_io.Data_IO_validation(args.region,
args.nlevs,
datasetfile,
args.locations['normaliser_loc'],
xvars=args.xvars,
yvars=args.yvars,
yvars2=args.yvars2,
add_adv=False,
no_norm=True)
print(args.xvars)
recursion_indx = list(range(args.nlevs))
x, y, y2, xmean, xstd, ymean, ystd, ymean2, ystd2 = nn_data.get_data()
# model = set_model(args)
x_split = nn_data.split_data(x, xyz='x')
yt_split = nn_data.split_data(y, xyz='y')
# yt_inverse = nn_data._inverse_transform(y,ymean,ystd)
yt_inverse = y
yt_inverse_split = nn_data.split_data(yt_inverse, xyz='y')
qnext_inv = yt_inverse_split['qtot'][1:]
tnext_inv = yt_inverse_split['theta'][1:]
# x_inv_split = nn_data._inverse_transform_split(x_split,xmean,xstd,xyz='x')
x_inv_split = nn_data.split_data(x, xyz='x')
qtot_inv = yt_inverse_split['qtot'][:-1]
theta_inv = yt_inverse_split['theta'][:-1]
output = []
output.append(qtot_inv[0])
toutput = []
toutput.append(theta_inv[0])
# xcopy = x.clone()
difflist = []
# args.xvar_multiplier = [1000., 1.]
for v, m in zip(args.xvars, args.xvar_multiplier):
vdiff = (x_split[v][1:] - x_split[v][:-1]) * m
# print(vdiff.shape)
difflist.append(vdiff[:-1])
xcopy = torch.cat(difflist, dim=1)
diff_output = []
qdiff = difflist[0]
diff_output.append((qdiff[0]))
tdiff_output = []
tdiff = difflist[1]
tdiff_output.append((tdiff[0]))
# plotX(x)
# sys.exit(0)
for t in range(1, len(qnext_inv) - 1):
# print(t)
yp_ = model(xcopy[t])
qnew = output[t - 1] + yp_[:args.nlevs] / 1000.
tnew = toutput[t - 1] + yp_[args.nlevs:] / 10.
print("tdiff", yp_[args.nlevs:])
print("qdiff", yp_[:args.nlevs])
negative_values = torch.where(qnew < 0.)
# print(negative_values[0])
if len(negative_values[0]) > 0:
qnew[negative_values] = 1.e-6 #output[t-1][negative_values]
# yp_[negative_values] = diff_output[t-1][negative_values]
output.append(qnew)
toutput.append(tnew)
diff_output.append(yp_[:args.nlevs])
tdiff_output.append(yp_[args.nlevs:])
# print("xcopy", xcopy[t,:(args.nlevs*2)])
# print("Diff predict", diff_output[t][0:10])
# print("Diff true", (qnext[t][0:10] - qnext[t-1][0:10])*1000.)
# print("Predict", output[t][:])
# print("True", qnext[t,:])
# yp = torch.from_numpy(np.concatenate([qnext_ml, tnext_ml], axis=1))
qp = torch.stack(output)
tp = torch.stack(toutput)
# yp = torch.from_numpy(qnext_ml)
# yp_inverse = nn_data._inverse_transform(yp, ymean, ystd)
# yp_inverse = yp
# yp_inverse_split = nn_data.split_data(yp_inverse, xyz='y')
# qnext_ml_inv = yp_inverse_split['qtot_next']
# tnext_ml_inv = yp_inverse_split['theta_next']
qnext_ml_inv = qp
tnext_ml_inv = tp
output = {
'qtot_next': qnext_inv.data.numpy(),
'qtot_next_ml': qnext_ml_inv.data.numpy(),
'qtot': qtot_inv.data.numpy(),
# 'qtot':qtot.data.numpy(),
# 'qtot_next':qnext.data.numpy(),
# 'qtot_next_ml':qnext_ml,
'theta_next': tnext_inv.data.numpy(),
'theta_next_ml': tnext_ml_inv.data.numpy()
}
hfilename = args.model_name.replace('.tar', '_scm.hdf5')
with h5py.File(hfilename, 'w') as hfile:
for k, v in output.items():
hfile.create_dataset(k, data=v)
def scm_diff_t_diag(model, datasetfile, args):
"""
SCM type run with qt_next prediction model
"""
nn_data = data_io.Data_IO_validation(args.region,
args.nlevs,
datasetfile,
args.locations['normaliser_loc'],
xvars=args.xvars,
yvars=args.yvars,
yvars2=args.yvars2,
add_adv=False,
no_norm=True)
print(args.xvars)
x, y, y2, xmean, xstd, ymean, ystd, ymean2, ystd2 = nn_data.get_data()
# model = set_model(args)
x_split = nn_data.split_data(x, xyz='x')
yt_split = nn_data.split_data(y, xyz='y')
# yt_inverse = nn_data._inverse_transform(y,ymean,ystd)
yt_inverse = y
yt_inverse_split = nn_data.split_data(yt_inverse, xyz='y')
tnext_inv = yt_inverse_split['theta'][1:]
theta_inv = yt_inverse_split['theta'][:-1]
toutput = []
toutput.append(theta_inv[0])
# xcopy = x.clone()
difflist = []
# args.xvar_multiplier = [1000., 1.]
for v, m in zip(args.xvars, args.xvar_multiplier):
vdiff = (x_split[v][1:] - x_split[v][:-1]) * m
# print(vdiff.shape)
difflist.append(vdiff[:-1])
xdiff = torch.cat(difflist, dim=1)
xcopy = xdiff.clone()
# plotX(xcopy[:10])
# sys.exit(0)
for t in range(1, len(tnext_inv) - 1):
# print(t)
# Difference prediction
# xcopy[t,:args.nlevs] = diff_output[t-1]
# xcopy[t,args.nlevs:] = tdiff[t-1]
yp_ = model(xcopy[t])
# yp = model(torch.rand(xcopy[t-1].shape)/100.)
# qnew = output[t-1] + yp_[:args.nlevs]/1000.
tnew = toutput[t - 1] + yp_[:args.nlevs] / 10.
# print("tdiff", yp_[:args.nlevs])
# print("x", xdiff[t,args.nlevs:args.nlevs*2])
# print("qdiff", yp_[:args.nlevs])
# negative_values = torch.where(qnew<0.)
# print(negative_values[0])
# if len(negative_values[0]) > 0:
# qnew[negative_values] = 1.e-6 #output[t-1][negative_values]
# yp_[negative_values] = diff_output[t-1][negative_values]
# output.append(qnew)
toutput.append(tnew)
# print("xcopy", xcopy[t,:args.nlevs])
# print("Diff predict", diff_output[t][0:10])
# print("Diff true", (qnext[t][0:10] - qnext[t-1][0:10])*1000.)
# print("Predict", output[t][:])
# print("True", tnext[t,:])
# yp = torch.from_numpy(np.concatenate([qnext_ml, tnext_ml], axis=1))
# qp = torch.stack(output)
tp = torch.stack(toutput)
# print(tnext_inv.data.numpy()[:100,0])
# print(tp[:200,0])
ml_diff = tnext_inv.data.numpy()[:200, 0] - tp.data.numpy()[:200, 0]
per_diff = theta_inv.data.numpy()[0, 0] - tnext_inv.data.numpy()[:200, 0]
tnext_ml_inv = tp
output = {
'theta': theta_inv.data.numpy(),
'theta_next': tnext_inv.data.numpy(),
'theta_next_ml': tnext_ml_inv.data.numpy()
}
hfilename = args.model_name.replace('.tar', '_scm.hdf5')
with h5py.File(hfilename, 'w') as hfile:
for k, v in output.items():
hfile.create_dataset(k, data=v)
def scm_diff_q_diag(model, datasetfile, args):
"""
SCM type run with qt_next prediction model
"""
nn_data = data_io.Data_IO_validation(args.region,
args.nlevs,
datasetfile,
args.locations['normaliser_loc'],
xvars=args.xvars,
yvars=args.yvars,
yvars2=args.yvars2,
add_adv=False,
no_norm=True)
print(args.xvars)
x, y, y2, xmean, xstd, ymean, ystd, ymean2, ystd2 = nn_data.get_data()
x_split = nn_data.split_data(x, xyz='x')
yt_split = nn_data.split_data(y, xyz='y')
yt_inverse = y
yt_inverse_split = nn_data.split_data(yt_inverse, xyz='y')
qnext_inv = yt_inverse_split['qtot'][1:]
qtot_inv = yt_inverse_split['qtot'][:-1]
output = []
output.append(qtot_inv[0])
difflist = []
for v, m in zip(args.xvars, args.xvar_multiplier):
vdiff = (x_split[v][1:] - x_split[v][:-1]) * m
difflist.append(vdiff[:-1])
xcopy = torch.cat(difflist, dim=1)
diff_output = []
# plotX(x)
# sys.exit(0)
for t in range(1, len(qnext_inv) - 1):
# Difference prediction
yp_ = model(xcopy[t])
qnew = output[t - 1] + yp_[:args.nlevs] / 1000.
negative_values = torch.where(qnew < 0.)
# print(negative_values[0])
if len(negative_values[0]) > 0:
qnew[negative_values] = 1.e-6 #output[t-1][negative_values]
# yp_[negative_values] = diff_output[t-1][negative_values]
output.append(qnew)
diff_output.append(yp_)
# xcopy[t,:args.nlevs] = yp_
# print("Diff predict", diff_output[t][0:10])
# print("Diff true", (qnext[t][0:10] - qnext[t-1][0:10])*1000.)
# print("Predict", output[t][:])
# print("True", qnext[t,:])
yp = torch.stack(output)
yp_inverse = yp
yp_inverse_split = nn_data.split_data(yp_inverse, xyz='y')
qnext_ml_inv = yp_inverse_split['qtot']
output = {
'qtot_next': qnext_inv.data.numpy(),
'qtot_next_ml': qnext_ml_inv.data.numpy(),
'qtot': qtot_inv.data.numpy(),
}
hfilename = args.model_name.replace('.tar', '_scm.hdf5')
with h5py.File(hfilename, 'w') as hfile:
for k, v in output.items():
hfile.create_dataset(k, data=v)
def scm(model, datasetfile, args):
"""
SCM type run with qt_next prediction model
"""
nn_data = data_io.Data_IO_validation(args.region,
args.nlevs,
datasetfile,
args.locations['normaliser_loc'],
xvars=args.xvars,
yvars=args.yvars,
yvars2=args.yvars2,
add_adv=False)
recursion_indx = list(range(args.nlevs))
x, y, y2, xmean, xstd, ymean, ystd, ymean2, ystd2 = nn_data.get_data()
# model = set_model(args)
x_split = nn_data.split_data(x, xyz='x')
yt_split = nn_data.split_data(y, xyz='y')
yt_inverse = nn_data._inverse_transform(y, ymean, ystd)
yt_inverse_split = nn_data.split_data(yt_inverse, xyz='y')
qnext = yt_split['qtot_next']
qnext_inv = yt_inverse_split['qtot_next']
x_inv_split = nn_data._inverse_transform_split(x_split,
xmean,
xstd,
xyz='x')
qtot_inv = x_inv_split['qtot']
output = []
output.append(qnext[0])
xcopy = x.clone()
for t in range(1, len(x) - 1):
# next tstep prediction
xcopy[t, recursion_indx] = output[t - 1]
output.append(model(xcopy[t]))
print("Predict", output[t][..., 0:10] - output[t - 1][..., 0:10])
print("True", qnext[t, 0:10] - qnext[t - 1, 0:10])
# Difference prediction
# xcopy[t,recursion_indx] = output[t-1]
# yt = model(xcopy[t])/100.
# output.append(yt + output[t-1])
# print("Predict", yt[...,0:10])
# print("True", qnext[t,0:10] - qnext[t-1,0:10])
# yp = torch.from_numpy(np.concatenate([qnext_ml, tnext_ml], axis=1))
yp = torch.stack(output)
# yp = torch.from_numpy(qnext_ml)
yp_inverse = nn_data._inverse_transform(yp, ymean, ystd)
# yp_inverse = yp
yp_inverse_split = nn_data.split_data(yp_inverse, xyz='y')
qnext_ml_inv = yp_inverse_split['qtot_next']
# tnext_ml_inv = yp_inverse_split['theta_next']
output = {
'qtot_next': qnext_inv.data.numpy(),
'qtot_next_ml': qnext_ml_inv.data.numpy(),
'qtot': qtot_inv.data.numpy(),
# 'qtot':qtot.data.numpy(),
# 'qtot_next':qnext.data.numpy(),
# 'qtot_next_ml':qnext_ml,
# 'theta_next':tnext_inv.data.numpy(),
# 'theta_next_ml':tnext_ml_inv.data.numpy()
}
hfilename = args.model_name.replace('.tar', '_scm.hdf5')
with h5py.File(hfilename, 'w') as hfile:
for k, v in output.items():
hfile.create_dataset(k, data=v)
