def build_monitor(model, path=''):
session = model.enquire_session()
global_step = mon.create_global_step(session)
# build monitor
print_task = mon.PrintTimingsTask().with_name('print')\
.with_condition(mon.PeriodicIterationCondition(500))\
.with_exit_condition(True)
sleep_task = mon.SleepTask(0.01).with_name('sleep').with_name('sleep')
saver_task = mon.CheckpointTask('./monitor-saves/' + path + model.name).with_name('saver')\
.with_condition(mon.PeriodicIterationCondition(500))\
.with_exit_condition(True)
file_writer = mon.LogdirWriter('./model-tensorboard/' + path + model.name)
model_tboard_task = mon.ModelToTensorBoardTask(
file_writer, model, only_scalars=False, parameters=list(model.trainable_parameters)).with_name('model_tboard')\
.with_condition(mon.PeriodicIterationCondition(50))\
.with_exit_condition(True)
lml_tboard_task = mon.LmlToTensorBoardTask(file_writer, model).with_name('lml_tboard')\
.with_condition(mon.PeriodicIterationCondition(50))\
.with_exit_condition(True)
monitor_tasks = [print_task, model_tboard_task, saver_task, sleep_task]
return monitor_tasks, session, global_step, file_writer
def task_factory(event_dir: str):
# create 2 extra summaries
dummy_vars = [tf.Variable(5.0), tf.Variable(6.0)]
dummy_vars_init = tf.variables_initializer(dummy_vars)
model.enquire_session().run(dummy_vars_init)
add_summaries = [tf.summary.scalar('dummy' + str(i), dummy_var)
for i, dummy_var in enumerate(dummy_vars)]
return mon.ModelToTensorBoardTask(event_dir, model, only_scalars=False,
additional_summaries=add_summaries)
def run_experiment(_seed, D, T, n_seq, batch_size, model_class,
optimizer, learning_rate, momentum, use_nesterov, maxiter, parallel_iterations,
process_noise_var, emission_noise_var,
Q_init_scale, R_init_scale, Ucov_chol_init_scale, X_chol_init_scale, kern_var_init, init_at_gp_on_Y,
train_Q, train_kern_var, train_Z, train_As, train_bs, train_Ss,
n_samples, n_ind_pts, base_save_path, init_from_disk,
test, filter_length, test_length, test_seq, test_samples, args):
tf.set_random_seed(_seed)
np.random.seed(_seed)
#tr = KinkTransitions(dim=D, Q=np.ones(D) * process_noise_var, name='KinkTransitions_datagen')
#SSM_datagen = SSM(X_init=np.zeros((T, D)), Y=np.zeros((T, D)), transitions=tr, emissions=em, name='SSM_datagen')
if args["emission_model"] == "gaussian":
em = GaussianEmissions(obs_dim=D, R=np.eye(D) * emission_noise_var, name='GaussianEmissions_datagen')
elif args["emission_model"] == "narendrali":
em = NarendraLiEmission(noise_var = emission_noise_var, name='NarendraLiEmission')
Y, U_0 = np.load(args["data"] + "Y_" +args["name"] +".npz"), np.load(args["data"] + "U_" +args["name"] +".npz")
Y, U = [np.transpose(Y[:,:,i]) for i in range(Y.shape[2])], None if U_0.shape[0] == 0 else [ np.transpose(U_0[:,:,i]) for i in range(U_0.shape[2])]
Xtest, Ytest, Utest = np.load(args["data"] + "Xtest_" +args["name"] +".npz"), np.load(args["data"] + "Ytest_" +args["name"] +".npz"), np.load(args["data"] + "Utest_" +args["name"] +".npz")
Xtest,Ytest, Utest = None if Xtest.shape[0]== 0 else [np.transpose(Xtest[:,:,i]) for i in range(Xtest.shape[2])], [np.transpose(Ytest[:,:,i]) for i in range(Ytest.shape[2])], None if U_0.shape[0] == 0 else [ np.transpose(Utest[:,:,i]) for i in range(Utest.shape[2])]
Xplot, Uplot = np.transpose(np.load(args["data"] + "Xplot_" +args["name"] +".npz")), np.transpose(np.load(args["data"] + "Uplot_" +args["name"] +".npz"))
Uplot = None if U_0.shape[0] == 0 else Uplot
Xplot = None if Xplot.shape[1]== 0 else Xplot # WE have already taken the transpose
Ypred, Upred = np.load(args["data"] + "Ypred_" +args["name"] +".npz"), np.load(args["data"] + "Upred_" +args["name"] +".npz")
Ypred, Upred = None if Ypred.shape[0]== 0 else [np.transpose(Ypred[:,:,i]) for i in range(Ypred.shape[2])], None if U_0.shape[0] == 0 else [ np.transpose(Upred[:,:,i]) for i in range(Upred.shape[2])]
Ypredseqahead, Upredseqahead = np.load(args["data"] + "Ypredseqahead_" +args["name"] +".npz"), np.load(args["data"] + "Upredseqahead_" +args["name"] +".npz")
Ypredseqahead, Upredseqahead = None if Ypredseqahead.shape[0]== 0 else [np.transpose(Ypredseqahead[:,:,i]) for i in range(Ypredseqahead.shape[2])], [None if U_0.shape[0] == 0 else np.transpose(Upredseqahead[:,:,i]) for i in range(Upredseqahead.shape[2])]
gaussian_GPSSM_name = 'SSM_SG_MultipleSequences'
model_classes = [gaussian_GPSSM_name, 'PRSSM_MS_MF', 'PRSSM_MS', 'GPSSM_MS_MF', 'GPSSM_MS_SPURIOUS',
'GPSSM_MS_FITC', 'GPSSM_MS_CUBIC', 'GPSSM_MS_VCDT', 'GPSSM_MS_FITC_SAMPLE_F', 'GPSSM_MS_FactorizedLinear', 'GPSSM_MS_FactorizedNonLinear']
if model_class in model_classes:
model_class_python = eval(model_class)
else:
raise ValueError('Unknown model class')
#kern = gp.kernels.Matern32(D, variance=kern_var_init, ARD=True, name='GPSSM_kern')
#Z = np.linspace(-8., 3., n_ind_pts)[:, None]
#mean_fn = mean_fns.Zero(D, name='GPSSM_mean_fn')
#Q_diag = np.ones(D) * Q_init_scale
#Ucov_chol = np.tile(np.eye(n_ind_pts)[None, ...], [D, 1, 1]) * Ucov_chol_init_scale
if init_from_disk is not None and init_from_disk[-4:] == '.npy':
GPSSM_params = np.load(init_from_disk).ravel()[0]
if 'GPSSM/Q_sqrt' in GPSSM_params.keys():
Q_diag = GPSSM_params['GPSSM/Q_sqrt'] ** 2.
if R_init_scale is not None:
em.Rchol = np.eye(D) * (R_init_scale ** 0.5)
em.compile()
extra_kwargs = {'batch_size': batch_size}
model = model_class_python(D,
Y,
inputs=U,
emissions=em,
n_ind_pts = n_ind_pts,
n_samples=n_samples,
seed=None,
name='GPSSM',
**extra_kwargs)
if args["emission_model"] == "gaussian":
if args["train_emission"]:
model.emissions.Rchol.trainable = False
model.emissions.C.trainable = False
model.emissions.bias.trainable = False
else:
model.emissions.Rchol.trainable = False
model.emissions.C.trainable = False
model.emissions.bias.trainable = False
elif args["emission_model"] == "narendrali":
model.emissions.trainable= args["train_emission"]
model.emissions.Rchol.trainable = False
model.emissions.C.trainable = False
model.emissions.bias.trainable = False
transitions = model
if train_As is not None: model.As.trainable = train_As
if train_bs is not None: model.bs.trainable = train_bs
if train_Ss is not None: model.S_chols.trainable = train_Ss
if model_class != 'GPSSM_MS_FactorizedLinear':
transitions.Q_sqrt.trainable = train_Q
transitions.Z.trainable = train_Z
try:
transitions.kern.kern.variance.trainable = train_kern_var
except:
warnings.warn('Could not set trainable status of the kernel\'s variance: default is trainable')
session = model.enquire_session()
if init_from_disk is not None:
if init_from_disk[-4:] != '.npy':
matching_dirs = glob(init_from_disk)
assert len(matching_dirs) == 1, 'More/Less than one matching run found'
matching_dir = matching_dirs[0]
checkpoint_loc = tf.train.latest_checkpoint(matching_dir + '/checkpoints')
tf.train.Saver().restore(session, checkpoint_loc)
GPSSM_params = model.read_values(session)
if 'PRSSM' in model_class:
offending_keys = [k for k in GPSSM_params.keys() if ('/As' in k or '/bs' in k or '/S_chols' in k)]
for k in offending_keys: GPSSM_params.pop(k)
model.assign(GPSSM_params)
model.compile()
print('================== Successfully initialised GPSSM params from: ' + init_from_disk + ' ==================')
print('Likelihood at initialisation loaded from disk:', model.compute_log_likelihood())
experiment_id = args["name"]
save_path = os.path.join(args["largestorage"], experiment_id)
if optimizer == 'adam':
optimizer = gp.train.AdamOptimizer(learning_rate=learning_rate)
elif optimizer == 'momentum':
optimizer = gp.train.MomentumOptimizer(learning_rate=learning_rate,
momentum=momentum, use_nesterov=use_nesterov)
else:
raise ValueError('Unknown optimizer')
global_step = mon.create_global_step(session)
print_task = mon.PrintTimingsTask() \
.with_name('print') \
.with_condition(mon.PeriodicIterationCondition(100))
checkpoint_task = mon.CheckpointTask(checkpoint_dir=save_path + '/checkpoints',
saver=tf.train.Saver(max_to_keep=100, save_relative_paths=True)) \
.with_name('checkpoint') \
.with_condition(mon.PeriodicIterationCondition(10000)) \
.with_exit_condition(True)
with mon.LogdirWriter(save_path + '/tensorboard') as writer:
tensorboard_task = mon.ModelToTensorBoardTask(writer, model, only_scalars=False) \
.with_name('tensorboard') \
.with_condition(mon.PeriodicIterationCondition(50)) \
.with_exit_condition(True)
monitor_tasks = [print_task, tensorboard_task, checkpoint_task]
with mon.Monitor(monitor_tasks, session, global_step, print_summary=True) as monitor:
optimizer.minimize(model, step_callback=monitor, global_step=global_step, maxiter=maxiter)
finallog = model.compute_log_likelihood()
if model_class in ['GPSSM_MS_FactorizedLinear', gaussian_GPSSM_name]:
#Z_Inds = model.transitions.Z.feat.Z.value
#Mu_Inds = model.transitions.Umu.value
#Sigma_Inds = model.transitions.Ucov_chol.value
X_samples = session.run(model._build_sample_qx())
else:
X_samples = session.run(model._build_sample()[0])
#Z_Inds = vcdt.Z.feat.Z.value
#Mu_Inds = model.Umu.value
#Sigma_Inds = model.Ucov_chol.value
#np.savez(os.path.join(save_path, 'LastState.npz'),
# Z_Inds=Z_Inds, Mu_Inds=Mu_Inds, Sigma_Inds=Sigma_Inds,log= finallog, X_Samples = X_samples)
np.savez(os.path.join(save_path, 'LastState.npz'),log= finallog, X_Samples = X_samples)
if Ypredseqahead is not None:
if model_class in ['GPSSM_MS_FactorizedLinear', gaussian_GPSSM_name]:
X_samples_filter = [np.transpose(xf, [1, 0, 2]) for xf in session.run(model._build_sample_qx())]
else:
X_samples_filter = session.run(model._build_sample()[0])
model_to_sample = model
X_samples, Y_samples, SE, RMSE, logp, NLPP, NLPP_mean, model_params = [], [], [], [], [], [], [], []
for i in range(len(Ypredseqahead)):
if model_class in ['GPSSM_MS_FactorizedLinear', gaussian_GPSSM_name]:
model_to_sample = GPSSM_Cubic(D, Ypredseqahead[i], emissions=model.emissions,inputs=Upredseqahead[i],
px1_mu=None, px1_cov=None,
kern=model.transitions.kern.kern,
Z=model.transitions.Z.feat.Z.value,
mean_fn=model.transitions.mean_fn,
Q_diag=model.transitions.Q_sqrt.value.copy() ** 2.,
Umu=model.transitions.Umu.value.copy(),
Ucov_chol=model.transitions.Ucov_chol.value.copy(),
qx1_mu=None, qx1_cov=None, As=None, bs=None, Ss=None,
n_samples=n_samples, seed=None,
name='GPSSM_posterior_sampling')
X_samples_i, Y_samples_i = model_to_sample.sample(T=test_length,
x0_samples=X_samples_filter[i][-1],
inputs=Upredseqahead[i],
cubic=False)
Ypred_i = Ypredseqahead[i]
SE_i = np.square(Y_samples_i.mean(1)[1:,:] - Ypred_i)
RMSE_i = np.sqrt(np.mean(np.sum(SE_i, -1)))
logp_i = session.run(
model_to_sample.emissions.logp(tf.constant(X_samples_i[1:,:]), tf.constant(Ypred_i[:, None, :])))
NLPP_i = - np.log(np.mean(np.exp(logp_i), axis=-1)) # We do not simply average over the logs, but ove the real
NLPP_mean_i = NLPP_i.mean()
print(RMSE_i, NLPP_mean_i)
X_samples.append(X_samples_i)
Y_samples.append(Y_samples_i)
SE.append(SE_i)
RMSE.append(RMSE_i)
logp.append(logp_i)
NLPP.append(NLPP_i)
NLPP_mean.append(NLPP_mean_i)
model_params.append(model.read_values(session=session))
np.savez(os.path.join(save_path, 'test_results2.npz'),
X_samples=X_samples, Y_samples=Y_samples, Ypred=Ypredseqahead,
SE=SE, RMSE=RMSE, logp=logp, NLPP=NLPP, NLPP_mean=NLPP_mean,
model_params=model_params, Upred = Upredseqahead)
pred_session = em.enquire_session()
pred_mu = []
pred_var = []
if Xtest is not None:
for i in range(len(Xtest)):
if Utest is not None:
Xin = np.concatenate([Xtest[i], Utest[i]], axis=1 )
else:
Xin = Xtest[i]
Xin = tf.constant(Xin)
if model_class == "GPSSM_MS_FactorizedLinear":
F_mu, F_var = pred_session.run(
model.transitions.conditional(Xin, add_noise=False))
else:
F_mu, F_var = session.run(model._build_predict_f(Xin))
pred_mu.extend(F_mu)
pred_var.extend(F_var)
np.save(args["dir"] + args["name"]+ "_pred", np.stack(np.stack(pred_mu), np.stack(pred_var)))
if Xplot is not None :
if Uplot is not None:
Xin = np.concatenate([Xplot, Uplot], axis=1 )
else:
Xin = Xplot
Xin = tf.constant(Xin)
if model_class == "GPSSM_MS_FactorizedLinear":
F_mu_plot, F_var_plot = pred_session.run(
model.transitions.conditional(Xin, add_noise=False))
else:
F_mu_plot, F_var_plot = session.run(model._build_predict_f(Xin))
np.save(args["dir"] + args["name"]+ "_plot", np.stack([F_mu_plot,F_var_plot]))
if Ypred is not None:
batch_size = None
Y_test_filter = [Ypred[i][:filter_length, :] for i in range(len(Ypred))]
U_test_filter = [Upred[i][:filter_length+1, :] for i in range(len(Ypred))]
modeltest = model_class_python(D, Y_test_filter, emissions=model.emissions, inputs= U_test_filter,
px1_mu=None, px1_cov=None,
kern=model.kern.kern,
Z=model.Z.feat.Z.value,
mean_fn=model.mean_fn,
Q_diag=model.Q_sqrt.value.copy() ** 2.,
Umu=model.Umu.value.copy(),
Ucov_chol=model.Ucov_chol.value.copy(),
qx1_mu=None, qx1_cov=None, As=None, bs=None, Ss=None,
n_samples=n_samples, batch_size=batch_size,
seed=None,
name='GPSSM_posterior')
modeltest.trainable = False
modeltest.qx1_mu.trainable = True
modeltest.qx1_cov_chol.trainable = True
if 'PRSSM' not in model_class:
modeltest.As.trainable = True
modeltest.bs.trainable = True
modeltest.S_chols.trainable = True
print('Likelihood at initialisation on filtering test sequences:', modeltest.compute_log_likelihood())
# We use the same optimizer again
global_step = mon.create_global_step(session)
print_task = mon.PrintTimingsTask() \
.with_name('print') \
.with_condition(mon.PeriodicIterationCondition(100))
checkpoint_task = mon.CheckpointTask(checkpoint_dir=save_path + '/checkpoints',
saver=tf.train.Saver(max_to_keep=100, save_relative_paths=True)) \
.with_name('checkpoint') \
.with_condition(mon.PeriodicIterationCondition(10000)) \
.with_exit_condition(True)
with mon.LogdirWriter(save_path + '/tensorboard') as writer:
tensorboard_task = mon.ModelToTensorBoardTask(writer, model, only_scalars=False) \
.with_name('tensorboard') \
.with_condition(mon.PeriodicIterationCondition(50)) \
.with_exit_condition(True)
monitor_tasks = [print_task, tensorboard_task, checkpoint_task]
with mon.Monitor(monitor_tasks, session, global_step, print_summary=True) as monitor:
optimizer.minimize(modeltest, step_callback=monitor, global_step=global_step, maxiter=maxiter)
if test_samples is not None:
modeltest.n_samples = test_samples
modeltest.compile()
if model_class in ['GPSSM_MS_FactorizedLinear', gaussian_GPSSM_name]:
X_samples_filter = [np.transpose(xf, [1, 0, 2]) for xf in session.run(modeltest._build_sample_qx())]
model_to_sample = GPSSM_Cubic(D, Y_test_filter, emissions=modeltest.emissions, inputs =U_test_filter,
px1_mu=None, px1_cov=None,
kern=modeltest.transitions.kern.kern,
Z=modeltest.transitions.Z.feat.Z.value,
mean_fn=modeltest.transitions.mean_fn,
Q_diag=modeltest.transitions.Q_sqrt.value.copy() ** 2.,
Umu=modeltest.transitions.Umu.value.copy(),
Ucov_chol=modeltest.transitions.Ucov_chol.value.copy(),
qx1_mu=None, qx1_cov=None, As=None, bs=None, Ss=None,
n_samples=n_samples, seed=None,
name='GPSSM_posterior_sampling')
else:
X_samples_filter = session.run(modeltest._build_sample()[0])
model_to_sample = modeltest
Y_samples_filter = [modeltest.emissions.sample_conditional(xf[1:,:]) for xf in X_samples_filter]
X_samples, Y_samples, SE, RMSE, logp, NLPP, NLPP_mean, model_params = [], [], [], [], [], [], [], []
for i in range(len(Ypred)):
X_samples_i, Y_samples_i = model_to_sample.sample(T=test_length,
x0_samples=X_samples_filter[i][-1],
inputs=Upred[filter_length:test_length+1, :],
cubic=False)
Ypred_i = Ypred[i][:(filter_length + test_length), :]
X_samples_i = np.concatenate([X_samples_filter[i], X_samples_i[1:]], 0)
Y_samples_i = np.concatenate([Y_samples_filter[i], Y_samples_i[1:]], 0)
SE_i = np.square(Y_samples_i.mean(1) - Ypred_i)
RMSE_i = np.sqrt(np.mean(np.sum(SE_i, -1)))
logp_i = session.run(
model_to_sample.emissions.logp(tf.constant(X_samples_i), tf.constant(Ypred_i[:, None, :])))
NLPP_i = - np.log(np.mean(np.exp(logp_i), axis=-1)) # We do not simply average over the logs, but ove the real
NLPP_mean_i = NLPP_i.mean()
print(RMSE_i, NLPP_mean_i)
X_samples.append(X_samples_i)
Y_samples.append(Y_samples_i)
SE.append(SE_i)
RMSE.append(RMSE_i)
logp.append(logp_i)
NLPP.append(NLPP_i)
NLPP_mean.append(NLPP_mean_i)
model_params.append(modeltest.read_values(session=session))
np.savez(os.path.join(save_path, 'test_results.npz'),
X_samples=X_samples, Y_samples=Y_samples, Ypred=Ypred, Upred= Upred,
SE=SE, RMSE=RMSE, logp=logp, NLPP=NLPP, NLPP_mean=NLPP_mean, filter_length=filter_length,
model_params=model_params)
def run_experiment(_seed, D, T, n_seq, batch_size, model_class, optimizer,
learning_rate, momentum, use_nesterov, maxiter,
parallel_iterations, process_noise_var, emission_noise_var,
Q_init_scale, R_init_scale, Ucov_chol_init_scale,
X_chol_init_scale, kern_var_init, init_at_gp_on_Y, train_R,
train_C, train_b, train_Q, train_kern_var, train_Z,
train_As, train_bs, train_Ss, n_samples, n_ind_pts,
base_save_path, init_from_disk, test, filter_length,
test_length, test_seq, test_samples):
tf.set_random_seed(_seed)
np.random.seed(_seed)
tr = KinkTransitions(dim=D,
Q=np.ones(D) * process_noise_var,
name='KinkTransitions_datagen')
em = GaussianEmissions(obs_dim=D,
R=np.eye(D) * emission_noise_var,
name='GaussianEmissions_datagen')
SSM_datagen = SSM(X_init=np.zeros((T, D)),
Y=np.zeros((T, D)),
transitions=tr,
emissions=em,
name='SSM_datagen')
X, Y = SSM_datagen.sample(T, N=n_seq)
X, Y = list(X), list(Y)
if test:
X_test, Y_test = SSM_datagen.sample(filter_length + test_length,
N=test_seq)
Y_test_filter = list(Y_test[:, :filter_length])
gaussian_GPSSM_name = 'SSM_nf_MULTISEQ'
model_classes = [
gaussian_GPSSM_name, 'PRSSM_MS_MF', 'PRSSM_MS', 'GPSSM_MS_MF',
'GPSSM_MS_SPURIOUS', 'GPSSM_MS_FITC', 'GPSSM_MS_CUBIC',
'GPSSM_MS_VCDT', 'GPSSM_MS_FITC_SAMPLE_F', 'GPSSM_VCDT_Stick_Land'
]
if model_class in model_classes:
model_class_python = eval(model_class)
else:
raise ValueError('Unknown model class')
kern = gp.kernels.RBF(D,
variance=kern_var_init,
ARD=True,
name='GPSSM_kern')
Z = np.linspace(-8., 3., n_ind_pts)[:, None]
mean_fn = mean_fns.Zero(D, name='GPSSM_mean_fn')
Q_diag = np.ones(D) * Q_init_scale
Ucov_chol = np.tile(np.eye(n_ind_pts)[None, ...],
[D, 1, 1]) * Ucov_chol_init_scale
if init_from_disk is not None and init_from_disk[-4:] == '.npy':
GPSSM_params = np.load(init_from_disk).ravel()[0]
if 'GPSSM/Q_sqrt' in GPSSM_params.keys():
Q_diag = GPSSM_params['GPSSM/Q_sqrt']**2.
elif init_at_gp_on_Y:
X_gp = np.concatenate([y[:-1] for y in Y])
Y_gp = np.concatenate([y[1:] for y in Y])
gpr = gp.models.SVGP(X_gp,
Y_gp,
kern,
gp.likelihoods.Gaussian(variance=Q_diag,
name='GPR_likelihood'),
Z=Z,
mean_function=mean_fn)
gpr.likelihood.trainable = train_Q
gpr.kern.variance.trainable = train_kern_var
gpr.feature.trainable = train_Z
opt = gp.train.ScipyOptimizer()
opt.minimize(gpr)
if R_init_scale is not None:
em.Rchol = np.eye(D) * (R_init_scale**0.5)
em.compile()
if model_class == 'GPSSM_VCDT_Stick_Land':
assert n_seq == 1
Y = Y[0]
extra_kwargs = {}
else:
extra_kwargs = {'batch_size': batch_size}
model = model_class_python(D,
Y,
emissions=em,
kern=kern,
Z=Z,
mean_fn=mean_fn,
Q_diag=Q_diag,
Ucov_chol=Ucov_chol,
n_samples=n_samples,
parallel_iterations=parallel_iterations,
seed=None,
name='GPSSM',
**extra_kwargs)
transitions = model
if train_As is not None: model.As.trainable = train_As
if train_bs is not None: model.bs.trainable = train_bs
if train_Ss is not None: model.S_chols.trainable = train_Ss
transitions.Q_sqrt.trainable = train_Q
try:
transitions.kern.kern.variance.trainable = train_kern_var
except:
warnings.warn(
'Could not set trainable status of the kernel\'s variance: default is trainable'
)
transitions.Z.trainable = train_Z
model.emissions.Rchol.trainable = train_R
model.emissions.C.trainable = train_C
model.emissions.bias.trainable = train_b
session_conf = tf.ConfigProto(intra_op_parallelism_threads=4,
inter_op_parallelism_threads=4)
session = model.enquire_session()
if test:
assert init_from_disk is not None, 'Have to initialise before testing'
if init_from_disk is not None:
if init_from_disk[-4:] != '.npy':
matching_dirs = glob(init_from_disk)
assert len(
matching_dirs) == 1, 'More/Less than one matching run found'
matching_dir = matching_dirs[0]
checkpoint_loc = tf.train.latest_checkpoint(matching_dir +
'/checkpoints')
tf.train.Saver().restore(session, checkpoint_loc)
GPSSM_params = model.read_values(session)
if 'PRSSM' in model_class:
offending_keys = [
k for k in GPSSM_params.keys()
if ('/As' in k or '/bs' in k or '/S_chols' in k)
]
for k in offending_keys:
GPSSM_params.pop(k)
model.assign(GPSSM_params)
model.compile()
print(
'================== Successfully initialised GPSSM params from: ' +
init_from_disk + ' ==================')
print('Likelihood at initialisation loaded from disk:',
model.compute_log_likelihood())
elif init_at_gp_on_Y:
transitions.Q_sqrt = gpr.likelihood.variance.value**0.5
if model_class != gaussian_GPSSM_name:
model.S_chols = model.S_chols.value * 0. + gpr.likelihood.variance.value**0.5
transitions.Umu = gpr.q_mu.value.T
transitions.Ucov_chol = gpr.q_sqrt.value
transitions.kern.kern.variance = gpr.kern.variance.value
transitions.kern.kern.lengthscales = gpr.kern.lengthscales.value
transitions.Z.feat.Z = gpr.feature.Z.value
transitions.mean_fn.assign(gpr.mean_function.read_values())
model.compile()
print(
'================== Successfully initialised using a GP fit on the observations =================='
)
print('Likelihood at initialisation from GP fit on the observations:',
model.compute_log_likelihood())
if test:
batch_size = None
model = model_class_python(D,
Y_test_filter,
emissions=model.emissions,
px1_mu=None,
px1_cov=None,
kern=model.kern.kern,
Z=model.Z.feat.Z.value,
mean_fn=model.mean_fn,
Q_diag=model.Q_sqrt.value.copy()**2.,
Umu=model.Umu.value.copy(),
Ucov_chol=model.Ucov_chol.value.copy(),
qx1_mu=None,
qx1_cov=None,
As=None,
bs=None,
Ss=None,
n_samples=n_samples,
batch_size=batch_size,
seed=None,
parallel_iterations=parallel_iterations,
name='GPSSM_posterior')
model.trainable = False
model.qx1_mu.trainable = True
model.qx1_cov_chol.trainable = True
if 'PRSSM' not in model_class:
model.As.trainable = True
model.bs.trainable = True
model.S_chols.trainable = True
print('Likelihood at initialisation on filtering test sequences:',
model.compute_log_likelihood())
# Monitoring:
experiment_id = model_class \
+ '__T_' + str(T) + '__n_seq_' + str(n_seq) \
+ '__Q_' + str(process_noise_var) + '__R_' + str(emission_noise_var) \
+ '__n_samples_' + str(n_samples) + '__M_' + str(n_ind_pts)
if test:
experiment_id += '__test'
if init_from_disk is not None:
experiment_id += '__initialised'
save_path = os.path.join(base_save_path,
experiment_id + '__' + str(datetime.now()))
if optimizer == 'adam':
optimizer = gp.train.AdamOptimizer(learning_rate=learning_rate)
elif optimizer == 'momentum':
optimizer = gp.train.MomentumOptimizer(learning_rate=learning_rate,
momentum=momentum,
use_nesterov=use_nesterov)
else:
raise ValueError('Unknown optimizer')
global_step = mon.create_global_step(session)
print_task = mon.PrintTimingsTask() \
.with_name('print') \
.with_condition(mon.PeriodicIterationCondition(100))
checkpoint_task = mon.CheckpointTask(checkpoint_dir=save_path + '/checkpoints',
saver=tf.train.Saver(max_to_keep=100, save_relative_paths=True)) \
.with_name('checkpoint') \
.with_condition(mon.PeriodicIterationCondition(10000)) \
.with_exit_condition(True)
with mon.LogdirWriter(save_path + '/tensorboard') as writer:
tensorboard_task = mon.ModelToTensorBoardTask(writer, model, only_scalars=False) \
.with_name('tensorboard') \
.with_condition(mon.PeriodicIterationCondition(50)) \
.with_exit_condition(True)
monitor_tasks = [print_task, tensorboard_task, checkpoint_task]
with mon.Monitor(monitor_tasks,
session,
global_step,
print_summary=True) as monitor:
optimizer.minimize(model,
step_callback=monitor,
global_step=global_step,
maxiter=maxiter)
if test:
if test_samples is not None:
model.n_samples = test_samples
model.compile()
if model_class == gaussian_GPSSM_name:
X_samples_filter = [
np.transpose(xf, [1, 0, 2])
for xf in session.run(model._build_sample_qx())
]
model_to_sample = GPSSM_CUBIC(
D,
Y_test[0],
emissions=model.emissions,
px1_mu=None,
px1_cov=None,
kern=model.transitions.kern.kern,
Z=model.transitions.Z.feat.Z.value,
mean_fn=model.transitions.mean_fn,
Q_diag=model.transitions.Q_sqrt.value.copy()**2.,
Umu=model.transitions.Umu.value.copy(),
Ucov_chol=model.transitions.Ucov_chol.value.copy(),
qx1_mu=None,
qx1_cov=None,
As=None,
bs=None,
Ss=None,
n_samples=n_samples,
seed=None,
parallel_iterations=parallel_iterations,
name='GPSSM_posterior_sampling')
else:
X_samples_filter = session.run(model._build_sample()[0])
model_to_sample = model
Y_samples_filter = [
model.emissions.sample_conditional(xf) for xf in X_samples_filter
]
X_samples, Y_samples, SE, RMSE, logp, NLPP, NLPP_mean, model_params = [], [], [], [], [], [], [], []
for i in range(len(Y_test)):
X_samples_i, Y_samples_i = model_to_sample.sample(
T=len(Y_test[i]) - filter_length,
x0_samples=X_samples_filter[i][-1],
inputs=None,
cubic=False)
X_samples_i = np.concatenate(
[X_samples_filter[i], X_samples_i[1:]], 0)
Y_samples_i = np.concatenate(
[Y_samples_filter[i], Y_samples_i[1:]], 0)
SE_i = np.square(Y_samples_i.mean(1) - Y_test[i])
RMSE_i = np.sqrt(np.mean(np.sum(SE_i, -1)))
logp_i = session.run(
model_to_sample.emissions.logp(
tf.constant(X_samples_i), tf.constant(Y_test[i][:,
None, :])))
NLPP_i = -log(
mean(exp(logp_i), axis=-1)
) # We do not simply average over the logs, but ove the real
NLPP_mean_i = NLPP_i.mean()
print(RMSE_i, NLPP_mean_i)
X_samples.append(X_samples_i)
Y_samples.append(Y_samples_i)
SE.append(SE_i)
RMSE.append(RMSE_i)
logp.append(logp_i)
NLPP.append(NLPP_i)
NLPP_mean.append(NLPP_mean_i)
model_params.append(model.read_values(session=session))
np.savez(os.path.join(matching_dir, 'test_results.npz'),
X_samples=X_samples,
Y_samples=Y_samples,
X_test=X_test,
Y_test=Y_test,
SE=SE,
RMSE=RMSE,
logp=logp,
NLPP=NLPP,
NLPP_mean=NLPP_mean,
filter_length=filter_length,
model_params=model_params)
maxiter=10000
exp_path="./exp/tmp-cc%d" % int(cc)
#exp_path="./exp/temp"
print_task = mon.PrintTimingsTask()\
.with_name('print')\
.with_condition(mon.PeriodicIterationCondition(10))\
checkpoint_task = mon.CheckpointTask(checkpoint_dir=exp_path)\
.with_name('checkpoint')\
.with_condition(mon.PeriodicIterationCondition(15))\
with mon.LogdirWriter(exp_path) as writer:
tensorboard_task = mon.ModelToTensorBoardTask(writer, model)\
.with_name('tensorboard')\
.with_condition(mon.PeriodicIterationCondition(100))\
.with_exit_condition(True)
monitor_tasks = [tensorboard_task] # [print_task, tensorboard_task]
with mon.Monitor(monitor_tasks, session, global_step, print_summary=True) as monitor:
model.layers[0].feature.Z.assign(Zcp.copy())
model.layers[0].kern.lengthscales.assign(np.ones((nodes, nodes)))
optimiser.minimize(model, step_callback=monitor, global_step=global_step, maxiter=maxiter)
#optimiser.minimize(model, step_callback=monitor, maxiter=maxiter)
res = []
for i in range(nodes):
for j in range(nodes):
res.append((i,j,model.layers[0].kern.lengthscales.value[j][i]))
def task_factory(event_dir: str):
return mon.ModelToTensorBoardTask(event_dir, model, only_scalars=True)
global_step = gpmon.create_global_step(session)
# create the gpmonitor tasks
print_task = gpmon.PrintTimingsTask().with_name('print') \
.with_condition(gpmon.PeriodicIterationCondition(monitor_lag)) \
.with_exit_condition(True)
savedir = os.path.join(root_savedir, 'monitor-saves')
saver_task = gpmon.CheckpointTask(savedir).with_name('saver') \
.with_condition(gpmon.PeriodicIterationCondition(save_lag)) \
.with_exit_condition(True)
file_writer = gpmon.LogdirWriter(root_logdir, session.graph)
model_tboard_task = gpmon.ModelToTensorBoardTask(file_writer, model).with_name('model_tboard') \
.with_condition(gpmon.PeriodicIterationCondition(monitor_lag)) \
.with_exit_condition(True)
train_tboard_task = LoglikelTensorBoardTask(file_writer, model, X_train, Y_train,
summary_name='train_ll').with_name('train_tboard') \
.with_condition(gpmon.PeriodicIterationCondition(monitor_lag)) \
.with_exit_condition(True)
# put the tasks together in a monitor
monitor_tasks = [print_task, model_tboard_task, train_tboard_task, saver_task]
# add one more if there is a validation set
if X_valid is not None:
test_tboard_task = LoglikelTensorBoardTask(file_writer, model, X_valid, Y_valid,
summary_name='test_ll').with_name('test_tboard') \
.with_condition(gpmon.PeriodicIterationCondition(monitor_lag)) \
def task_factory(writer: mon.LogdirWriter):
return mon.ModelToTensorBoardTask(writer, model, only_scalars=True)
saving_freq = 500
tensorboard_freq = 500
print_task = mon.PrintTimingsTask() \
.with_name('print') \
.with_condition(mon.PeriodicIterationCondition(print_freq))
saver = tf.train.Saver(max_to_keep=1, save_relative_paths=True)
checkpoint_task = mon.CheckpointTask(checkpoint_dir=checkpoint_path, saver=saver) \
.with_name('checkpoint') \
.with_condition(mon.PeriodicIterationCondition(saving_freq)) \
.with_exit_condition(True)
writer = mon.LogdirWriter(tensorboard_path)
tensorboard_task = mon.ModelToTensorBoardTask(writer, model) \
.with_name('tensorboard') \
.with_condition(mon.PeriodicIterationCondition(tensorboard_freq))
monitor_tasks = [print_task, tensorboard_task, checkpoint_task]
#################################### training
with mon.Monitor(monitor_tasks, sess, model.global_step,
print_summary=True) as monitor:
try:
mon.restore_session(sess, checkpoint_path)
except ValueError:
pass
iterations_to_go = max([ARGS.iterations - sess.run(model.global_step), 0])
def ex1():
fX_dim = 1
M = 100
X, Y, Xt, Yt = LoadData.load_ocean()
# annoyingly only float32 and lower is supported by the conv layers
f = lambda x: tf.cast(NN.cnn_fn(tf.cast(x, tf.float32), fX_dim), float_type)
kern = NNComposedKernel(gpflow.kernels.Matern32(fX_dim), f)
# build the model
lik = gpflow.likelihoods.Gaussian()
Z = kmeans2(X, M, minit='points')[0]
model = NN_SVGP(X, Y, kern, lik, Z=Z, minibatch_size=200)
session = model.enquire_session()
global_step = mon.create_global_step(session)
# print
print_task = mon.PrintTimingsTask().with_name('print') \
.with_condition(mon.PeriodicIterationCondition(10)) \
.with_exit_condition(True)
sleep_task = mon.SleepTask(0.01).with_name('sleep').with_name('sleep')
saver_task = mon.CheckpointTask('./monitor-saves').with_name('saver') \
.with_condition(mon.PeriodicIterationCondition(10)) \
.with_exit_condition(True)
file_writer = mon.LogdirWriter('./model-tensorboard')
model_tboard_task = mon.ModelToTensorBoardTask(file_writer, model).with_name('model_tboard') \
.with_condition(mon.PeriodicIterationCondition(10)) \
.with_exit_condition(True)
lml_tboard_task = mon.LmlToTensorBoardTask(file_writer, model).with_name('lml_tboard') \
.with_condition(mon.PeriodicIterationCondition(100)) \
.with_exit_condition(True)
custom_tboard_task = CustomTensorBoardTask(file_writer, model, Xt, Yt).with_name('custom_tboard') \
.with_condition(mon.PeriodicIterationCondition(100)) \
.with_exit_condition(True)
monitor_tasks = [print_task, model_tboard_task, lml_tboard_task, custom_tboard_task, saver_task, sleep_task]
monitor = mon.Monitor(monitor_tasks, session, global_step)
if os.path.isdir('./monitor-saves'):
mon.restore_session(session, './monitor-saves')
# use gpflow wrappers to train. NB all session handling is done for us
optimiser = gpflow.training.AdamOptimizer(0.001)
with mon.Monitor(monitor_tasks, session, global_step, print_summary=True) as monitor:
optimiser.minimize(model, step_callback=monitor, maxiter=30000, global_step=global_step)
file_writer.close()
print('LML after the optimisation: %f' % m.compute_log_likelihood())
# # predictions
pY, pYv = model.predict_y(Xt)
rmse = np.mean((pY - Yt) ** 2.0) ** 0.5
nlpp = -np.mean(-0.5 * np.log(2 * np.pi * pYv) - 0.5 * (Yt - pY) ** 2.0 / pYv)
print('rmse is {:.4f}%, nlpp is {:.f}%'.format(rmse, nlpp))