def expected_PosteriormeanVariance(self, X, L, hyperpars, K_expected):
size = self.n_tasks * self.n_train
beta, varm, loc, varc = hyperpars
Cov_test_expected = mixing_Covariance_diag(K_expected, self.Wmix, varm,
varc) # shape M
#----- covariance between test data and simulationn training data
Kx3 = self.kernel(self.Xtrain, X, beta) # shape Q x Ntrain x Nb
Kx3_expected = tf.reduce_mean(Kx3, axis=-1,
keepdims=True) # shape Q x Ntrain x 1
Cov_mixed_expected = mixing_Covariance(
Kx3_expected, self.Wmix, varm, varc) # shape M x N_train x M x 1
Cov_mixed_expected = tf.reshape(Cov_mixed_expected,
[size, self.n_tasks])
mean_training = tf.tile(loc[:, tf.newaxis], [1, self.n_train])
mean_training = tf.reshape(mean_training, [size, 1])
Y = tf.transpose(self.Ytrain)
Y = tf.reshape(Y, [size, 1]) - mean_training
mean, var = posterior_Gaussian(L, Cov_mixed_expected,
Cov_test_expected, Y, False)
var = tf.maximum(var, 1e-40)
mean = mean + loc[:, tf.newaxis]
mean_and_var = tf.concat([mean, var], axis=1)
return mean_and_var
def full_PosteriormeanVariance(self, X, L, hyperpars):
n_new = X.shape[0].value
size_new = self.n_tasks * n_new
beta, varm, loc, varc = hyperpars
Cov_test = varc * tf.reduce_sum(tf.square(self.Wmix),
axis=1) + tf.reduce_sum(varm, axis=1)
Cov_test = tf.tile(Cov_test[:, tf.newaxis], [1, n_new])
Cov_test = tf.reshape(Cov_test, [size_new])
Kx3 = self.kernel(self.Xtrain, X, beta)
size = self.n_tasks * self.n_train
Cov_mixed = mixing_Covariance(Kx3, self.Wmix, varm,
varc) # with shape M x N x M x N_test
Cov_mixed = tf.reshape(Cov_mixed, [size, size_new])
mean_training = tf.tile(loc[:, tf.newaxis], [1, self.n_train])
mean_training = tf.reshape(mean_training, [size, 1])
mean_test = tf.tile(loc[:, tf.newaxis], [1, n_new])
mean_test = tf.reshape(mean_test, [size_new, 1])
Y = tf.transpose(self.Ytrain)
Y = tf.reshape(Y, [size, 1]) - mean_training
mean, var = posterior_Gaussian(L, Cov_mixed, Cov_test, Y, False)
mean = mean + mean_test
mean_and_var = tf.concat([mean, var], axis=1)
return mean_and_var
def predict_posterior(self, Xnew, hyperpar_samples):
# function used to compute the mean and variance of posterior Gaussian process
# Inputs:
# Xnew := 2-dimensional numpy array containing the input samples
# hyperpar_samples := list (of the form [loc_samples, varm_samples, beta_samples]) of numpy arrays containing samples for the hyperparameters
Wmix = self.Wmix
noise = self.noise
noise_matrix = tf.tile(self.noise[:, tf.newaxis], [1, self.n_train])
noise_matrix = tf.linalg.diag(tf.reshape(noise_matrix, [-1]))
X = tf.convert_to_tensor(Xnew, tf.float32)
loc_samples, varm_samples, beta_samples, varc_samples = hyperpar_samples
beta = tf.convert_to_tensor(np.median(beta_samples, axis=0),
tf.float32)
varm = tf.convert_to_tensor(np.median(varm_samples, axis=0),
tf.float32)
loc = tf.convert_to_tensor(np.median(loc_samples, axis=0), tf.float32)
varc = tf.convert_to_tensor(np.median(varc_samples, axis=0),
tf.float32)
hyperpars = [beta, varm, loc, varc]
# ------- generate covariance matrix for training data and computing the corresponding cholesky factor
Kxx = self.kernel(self.Xtrain, self.Xtrain,
beta) # kernels for the latent Gaussian processes
# Kxx has shape R x N x N
Cov_train = mixing_Covariance(Kxx, Wmix, varm,
varc) # with shape M x N x M x N
size = self.n_tasks * self.n_train
Cov_train = tf.reshape(
Cov_train,
[size, size]) + noise_matrix + (self.jitter_level) * tf.eye(size)
#-------- Computing the cholesky factor ------------
L = tf.linalg.cholesky(Cov_train)
results = self.full_PosteriormeanVariance(X, L, hyperpars)
with tf.Session() as sess:
results_ = sess.run(results)
return results_
def joint_log_prob(self, noise, Wmix, beta, varm, loc, varc):
# function for computing the joint_log_prob given values for the inverse lengthscales
# the variance of the kernel and the mean of the simulator Gaussian process
#------ forming the kernels -----------------
Kxx = self.kernel(self.Xtrain, self.Xtrain, beta) # shape Q x N x N
Cov_train = mixing_Covariance(Kxx, Wmix, varm,
varc) # with shape M x N x M x N
size = self.n_tasks * self.n_train
noise_matrix = tf.tile(self.noise[:, tf.newaxis], [1, self.n_train])
noise_matrix = tf.linalg.diag(tf.reshape(noise_matrix, [-1]))
Cov_train = tf.reshape(
Cov_train,
[size, size]) + noise_matrix + self.jitter_level * tf.eye(size)
#-------- Computing the cholesky factor ------------
L = tf.linalg.cholesky(Cov_train)
#---- Multivariate normal random variable for the combined outputs -----
mean = tf.tile(loc[:, tf.newaxis], [1, self.n_train])
mean = tf.reshape(mean, [size])
rv_observations = tfd.MultivariateNormalTriL(loc=mean, scale_tril=L)
Yreshaped = tf.reshape(tf.transpose(self.Ytrain), [size])
#--- Collecting the log_probs from the different random variables
sum_log_prob = (rv_observations.log_prob(Yreshaped) +
self.rv_beta.log_prob(beta) +
self.rv_varm.log_prob(varm) +
self.rv_loc.log_prob(loc) +
self.rv_varc.log_prob(varc))
return sum_log_prob
def expected_predict_posterior(self, sampling_dict, hyperpar_samples):
# function used to compute the mean and variance of main effect Gaussian processes
# These are Gaussian processes of the form
# E[Y|X_i]
# This means that we are keeping a set of variables fixed (in this case the
# subset X_i) while averaging out over the rest of the variables. For simplicity,
# the variables are assumed to have uniform distributions. The integrals involved
# in the computation are approximated with Monte Carlo integration
# Inputs:
# sampling_dict := 4-dimensional numpy array containing the input samples
# hyperpar_samples := list (of the form [loc_samples, varm_samples, beta_samples]) of numpy arrays containing samples for the hyperparameters
loc_samples, varm_samples, beta_samples, varc_samples = hyperpar_samples
beta_median = np.median(beta_samples, axis=0)
varm_median = np.median(varm_samples, axis=0)
loc_median = np.median(loc_samples, axis=0)
varc_median = np.median(varc_samples, axis=0)
beta = tf.convert_to_tensor(beta_median, tf.float32)
varm = tf.convert_to_tensor(varm_median, tf.float32)
loc = tf.convert_to_tensor(loc_median, tf.float32)
varc = tf.convert_to_tensor(varc_median, tf.float32)
hyperpars = [beta, varm, loc, varc]
Wmix = self.Wmix
noise = self.noise
# ------- generate covariance matrix for training data and computing the corresponding cholesky factor
Kxx = self.kernel(self.Xtrain, self.Xtrain,
beta) # kernels for the latent Gaussian processes
# Kxx has shape R x N x N
Cov_train = mixing_Covariance(Kxx, Wmix, varm,
varc) # with shape M x N x M x N
size = self.n_tasks * self.n_train
noise_matrix = tf.tile(self.noise[:, tf.newaxis], [1, self.n_train])
noise_matrix = tf.linalg.diag(tf.reshape(noise_matrix, [-1]))
Cov_train = tf.reshape(
Cov_train,
[size, size]) + noise_matrix + (self.jitter_level) * tf.eye(size)
#-------- Computing the cholesky factor ------------
L = tf.linalg.cholesky(Cov_train)
K_expected = tf.Variable(tf.ones(self.n_latent), name='K_expected')
f = lambda Xin: self.expected_PosteriormeanVariance(
Xin, L, hyperpars, K_expected)
k = list(sampling_dict.keys())[0]
grid_points, num_samples1, _ = sampling_dict[k]['X_sampling'].shape
num_samples2, _ = sampling_dict[k]['diff_samples'].shape
n_input = self.dim_input
indices = set([i for i in range(n_input)])
beta_slice = tf.placeholder(tf.float32,
shape=[self.n_latent, n_input],
name='beta_slice')
diff_samples = tf.placeholder(tf.float32,
shape=[num_samples2, n_input],
name='diff_samples')
Xin = tf.placeholder(tf.float32,
shape=[grid_points, num_samples1, n_input],
name='Xin')
K_expected_new = self.expected_kernel(diff_samples, beta_slice)
K_expected_update = K_expected.assign(K_expected_new)
with tf.control_dependencies([K_expected_update]):
results = tf.map_fn(f, Xin)
collect_results = {}
init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
for i in sampling_dict.keys():
ids = sampling_dict[i]['fixed_indices_list']
ids_left = list(indices - set(ids))
ids_left.sort()
pad_size = len(ids)
diff_samples_batch = sampling_dict[i]['diff_samples']
diff_samples_batch = np.pad(diff_samples_batch,
((0, 0), (0, pad_size)),
mode='constant')
Xbatch = sampling_dict[i]['X_sampling']
beta_input = beta_median[:, ids_left]
beta_input = np.pad(beta_input, ((0, 0), (0, pad_size)),
mode='constant')
_, collect_results[i] = sess.run(
[K_expected_update, results],
feed_dict={
beta_slice: beta_input,
diff_samples: diff_samples_batch,
Xin: Xbatch
})
return collect_results
def posteriormeanVariance(self, Xtest, hyperpars, fullCov=False):
# generate posterior mean and variance for the Gaussian process, given values for the hyperparameters
# Xtest := N x D tensorflow array of new inputs
# output := mean of posterior distribution in the form of a N x Q array
# and (Co)variance of posterior in the form of a N x N X Q array if
# fullCov = True or a N x 1 array if fullCov = False
beta, varm, loc, varc = hyperpars
noise = self.noise
Wmix = self.Wmix
# ------- generate covariance matrix for training data and computing the corresponding cholesky factor
Kxx = self.kernel(self.Xtrain, self.Xtrain,
beta) # kernels for the latent Gaussian processes
# Kxx has shape R x N x N
Cov_train = mixing_Covariance(Kxx, Wmix, varm,
varc) # with shape M x N x M x N
size = self.n_tasks * self.n_train
noise_matrix = tf.tile(self.noise[:, tf.newaxis], [1, self.n_train])
noise_matrix = tf.linalg.diag(tf.reshape(noise_matrix, [-1]))
Cov_train = tf.reshape(
Cov_train,
[size, size]) + noise_matrix + (self.jitter_level) * tf.eye(size)
#-------- Computing the cholesky factor ------------
L = tf.linalg.cholesky(Cov_train)
#-------- generate covariance matrix for test data
n_test = Xtest.shape[0].value
size_test = self.n_tasks * n_test
if fullCov:
Kx2 = self.kernel(Xtest, Xtest, beta)
Cov_test = mixing_Covariance(Kx2, Wmix, varm, varc)
Cov_test = tf.reshape(
Cov_test, [size_test, size_test
]) + (noise + self.jitter_level) * tf.eye(size_test)
else:
Cov_test = varc * tf.reduce_sum(
tf.square(Wmix), axis=1) + tf.reduce_sum(varm, axis=1)
Cov_test = tf.tile(Cov_test[:, tf.newaxis], [1, n_test])
Cov_test = tf.reshape(Cov_test, [size_test])
#------- covariance between test data and training data
Kx3 = self.kernel(self.Xtrain, Xtest, beta)
Cov_mixed = mixing_Covariance(Kx3, Wmix, varm,
varc) # with shape M x N x M x N_test
Cov_mixed = tf.reshape(Cov_mixed, [size, size_test])
mean_training = tf.tile(loc[:, tf.newaxis], [1, self.n_train])
mean_training = tf.reshape(mean_training, [size, 1])
mean_test = tf.tile(loc[:, tf.newaxis], [1, n_test])
mean_test = tf.reshape(mean_test, [size_test, 1])
Y = tf.transpose(self.Ytrain)
Y = tf.reshape(Y, [size, 1]) - mean_training
mean_pos, var_pos = posterior_Gaussian(L, Cov_mixed, Cov_test, Y,
fullCov)
mean_pos = mean_pos + mean_test
return mean_pos, var_pos