def _random_scale_tril(self, event_size): n = np.int32(event_size * (event_size + 1) // 2) p = 2. * self._rng.random_sample(n).astype(np.float32) - 1. return distribution_util.fill_triangular(0.25 * p)
def _forward(self, x): return dist_util.fill_triangular(x, upper=self._upper)
def _forward(self, x):
return dist_util.fill_triangular(x, upper=self._upper)
def _random_scale_tril(self, event_size):
n = np.int32(event_size * (event_size + 1) // 2)
p = 2. * self._rng.random_sample(n).astype(np.float32) - 1.
return distribution_util.fill_triangular(0.25 * p)
def dgpr_variational_family(X,
Z,
Zm=None,
ls=1.,
kernel_func=kernel_util.rbf,
ridge_factor=1e-3,
name="",
**kwargs):
"""Defines the decoupled GP variational family for GPR.
Args:
X: (np.ndarray of float32) input training features, with dimension (Nx, D).
Z: (np.ndarray of float32) inducing points, shape (Ns, D).
Zm: (np.ndarray of float32 or None) inducing points for mean, shape (Nm, D).
If None then same as Z
ls: (float32) length scale parameter.
kernel_func: (function) kernel function.
ridge_factor: (float32) small ridge factor to stabilize Cholesky decomposition
mfvi_mixture: (float32) Whether to output variational family with a
mixture of MFVI.
n_mixture: (int) Number of MFVI mixture component to add.
name: (str) name for the variational parameter/random variables.
kwargs: Dict of other keyword variables.
For compatibility purpose with other variational family.
Returns:
q_f, q_sig: (ed.RandomVariable) variational family.
q_f_mean, q_f_sdev: (tf.Variable) variational parameters for q_f
"""
X = tf.convert_to_tensor(X)
Zs = tf.convert_to_tensor(Z)
Zm = tf.convert_to_tensor(Zm) if Zm is not None else Zs
Nx, Nm, Ns = X.shape.as_list()[0], Zm.shape.as_list()[0], Zs.shape.as_list(
)[0]
# 1. Prepare constants
# compute matrix constants
Kxx = kernel_func(X, ls=ls)
Kmm = kernel_func(Zm, ls=ls)
Kxm = kernel_func(X, Zm, ls=ls)
Kxs = kernel_func(X, Zs, ls=ls)
Kss = kernel_func(Zs, ls=ls, ridge_factor=ridge_factor)
# 2. Define variational parameters
# define free parameters (i.e. mean and full covariance of f_latent)
m = tf.get_variable(shape=[Nm, 1], name='{}_mean_latent'.format(name))
s = tf.get_variable(shape=[Ns * (Ns + 1) / 2],
name='{}_cov_latent_s'.format(name))
L = fill_triangular(s, name='{}_cov_latent_chol'.format(name))
# components for KL objective
H = tf.eye(Ns) + tf.matmul(L, tf.matmul(Kss, L), transpose_a=True)
cond_cov_inv = tf.matmul(L, tf.matrix_solve(H, tf.transpose(L)))
func_norm_mm = tf.matmul(m, tf.matmul(Kmm, m), transpose_a=True)
log_det_ss = tf.log(tf.matrix_determinant(H))
cond_norm_ss = tf.reduce_sum(tf.multiply(Kss, cond_cov_inv))
# compute sparse gp variational parameter (i.e. mean and covariance of P(f_obs | f_latent))
qf_mean = tf.squeeze(tf.tensordot(Kxm, m, [[1], [0]]),
name='{}_mean'.format(name))
qf_cov = (Kxx -
tf.matmul(Kxs, tf.matmul(cond_cov_inv, Kxs, transpose_b=True)) +
ridge_factor * tf.eye(Nx, dtype=tf.float32))
# 3. Define variational family
q_f = dist_util.VariationalGaussianProcessDecoupled(
loc=qf_mean,
covariance_matrix=qf_cov,
func_norm_mm=func_norm_mm,
log_det_ss=log_det_ss,
cond_norm_ss=cond_norm_ss,
name=name)
return q_f, qf_mean, qf_cov
def _initialize_variables(self,
use_single_gp: bool,
diagonal_wiener_process: bool = False,
single_variable_wiener_process: bool = False,
initial_gmat_value: float = None) -> None:
"""
Initialize the variance of the log-likelihood of the GP as a TensorFlow
variable. A logarithm-exponential transformation is used to ensure
positivity during optimization.
:param use_single_gp: boolean, indicates whether to use a single set of
hyperparameters for all states (useful for extremely scarce data
setting);
:param diagonal_wiener_process: use a diagonal Wiener process (in which
every term on the diagonal is a separate optimization variable);
:param single_variable_wiener_process: use a diagonal Wiener process in
which all the elements on the diagonal are equal to each other
(e.g. variable * I);
:param initial_gmat_value: initial value for the matrix G multiplicating
the Wiener process.
"""
with tf.variable_scope('gaussian_process'):
# Log-likelihood variances
if use_single_gp:
self.likelihood_logvariance = tf.Variable(
np.log(1.0),
dtype=tf.float64,
trainable=True,
name='variance_loglik')
self.likelihood_logvariances =\
self.likelihood_logvariance * tf.ones([self.dimensionality,
1, 1],
dtype=tf.float64)
else:
self.likelihood_logvariances = tf.Variable(
np.log(1.0) *
tf.ones([self.dimensionality, 1, 1], dtype=tf.float64),
dtype=tf.float64,
trainable=True,
name='variances_loglik')
self.likelihood_variances = tf.exp(self.likelihood_logvariances)
# G matrix
if single_variable_wiener_process:
g_vector_np = np.array([np.log(1.0)]).reshape(1, 1)
self.g_vector = tf.Variable(g_vector_np,
dtype=tf.float64,
name='g_vector',
trainable=True)
self.g_matrix = tf.exp(self.g_vector)\
* tf.eye(self.dimensionality_int, dtype=tf.float64)
return
elif diagonal_wiener_process:
n_elements_sigma = self.dimensionality_int
else:
n_elements_sigma = int(self.dimensionality_int *
(self.dimensionality_int + 1) / 2)
if initial_gmat_value:
self.g_vector = tf.Variable(
np.log(initial_gmat_value) *
tf.ones([n_elements_sigma], dtype=tf.float64),
name='g_vector',
trainable=True,
dtype=tf.float64)
else:
self.g_vector = tf.Variable(tf.random_normal(
[n_elements_sigma],
mean=np.log(0.1),
stddev=0.1,
dtype=tf.float64),
name='g_vector',
trainable=True,
dtype=tf.float64)
self.g_matrix = tf.diag(
tf.exp(self.g_vector[0:self.dimensionality]))
if initial_gmat_value:
self.g_matrix = self.g_matrix \
* tf.eye(self.dimensionality, dtype=tf.float64)
if not diagonal_wiener_process:
paddings = tf.constant([[1, 0], [0, 1]])
self.g_matrix += tf.pad(
fill_triangular(
(self.g_vector[self.dimensionality:n_elements_sigma])),
paddings)
return
def __init__(self, v, observation_space, action_space, learning_rate, batch_normalize, gamma, tau, epsilon, hidden_size, hidden_n, hidden_activation, batch_size, memory_capacity, load_path, covariance):
self.v = v
self.memory = Memory(memory_capacity,batch_size,v)
self.observation_space = observation_space
self.action_space = action_space
self.state_n = observation_space.shape[0]
self.action_n = action_space.shape[0]
self.learning_rate = learning_rate
self.gamma = gamma
self.tau = tau
self.epsilon = epsilon
self.resets = 0
H_layer_n = hidden_n
H_n = hidden_size
M_n = int((self.action_n)*(self.action_n+1)/2)
V_n = 1
mu_n = self.action_n
tf.reset_default_graph()
#neural network architecture
self.x = tf.placeholder(shape=[None,self.state_n], dtype=tf.float32, name="state")
self.u = tf.placeholder(shape=[None,self.action_n], dtype=tf.float32, name="action")
self.target = tf.placeholder(shape=[None,1], dtype=tf.float32, name="target")
self.H = Layer(self.x, H_n, activation=hidden_activation, batch_normalize=batch_normalize)
self.t_H = Layer(self.x, H_n, activation=hidden_activation, batch_normalize=batch_normalize) #target
self.updates = self.t_H.construct_update(self.H, self.tau)
for i in range(1,H_layer_n):
self.H = Layer(self.H.h, H_n, activation=hidden_activation, batch_normalize=batch_normalize)
self.t_H = Layer(self.t_H.h, H_n, activation=hidden_activation, batch_normalize=batch_normalize) #target
self.updates += self.t_H.construct_update(self.H, self.tau)
self.V = Layer(self.H.h, V_n, batch_normalize=batch_normalize)
self.t_V = Layer(self.t_H.h, V_n, batch_normalize=batch_normalize) #target
self.updates += self.t_V.construct_update(self.V, self.tau)
self.mu = Layer(self.H.h, mu_n, activation=tf.nn.tanh, batch_normalize=batch_normalize)
if covariance == "identity": #identity covariance
self.P = tf.eye(self.action_n,batch_shape=[tf.shape(self.x)[0]]) #identity covariance
elif covariance == "diagonal": #diagonal covariance with nn inputs
self.O = Layer(self.H.h, mu_n, batch_normalize=batch_normalize) #nn input to diagonal covariance
self.P = tf.matrix_set_diag(tf.eye(self.action_n,batch_shape=[tf.shape(self.x)[0]]), self.O.h) #diagonal covariance
else: #original NAF covariance
self.M = Layer(self.H.h, M_n, activation=tf.nn.tanh, batch_normalize=batch_normalize)
self.N = fill_triangular(self.M.h)
self.L = tf.matrix_set_diag(self.N, tf.exp(tf.matrix_diag_part(self.N)))
self.P = tf.matmul(self.L, tf.matrix_transpose(self.L)) #original NAF covariance
#self.P_inverse = tf.matrix_inverse(self.P) #precision matrix for exploration policy
self.D = tf.reshape(self.u - self.mu.h, [-1,1,self.action_n])
self.A = (-1.0/2.0)*tf.reshape(tf.matmul(tf.matmul(self.D, self.P), tf.transpose(self.D, perm=[0,2,1])), [-1,1]) #advantage function
self.Q = self.A + self.V.h
self.loss = tf.reduce_sum(tf.square(self.target - self.Q))
self.optimiser = tf.train.AdamOptimizer(learning_rate=self.learning_rate).minimize(self.loss)
self.sess = tf.Session()
init = tf.global_variables_initializer()
self.sess.run(init)
self.saver = tf.train.Saver()
if load_path is not None:
self.saver.restore(self.sess, load_path)
def variational_dgpr(X,
Zm,
Zs,
ls=1.,
kern_func=rbf,
ridge_factor=1e-3,
mfvi_mixture=False,
n_mixture=1):
"""Defines the mean-field variational family for GPR.
Args:
X: (np.ndarray of float32) input training features, with dimension (Nx, D).
Zm: (np.ndarray of float32) inducing points for mean, shape (Nm, D).
Zs: (np.ndarray of float32) inducing points for covar, shape (Ns, D).
ls: (float32) length scale parameter.
kern_func: (function) kernel function.
ridge_factor: (float32) small ridge factor to stabilize Cholesky decomposition
mfvi_mixture: (float32) Whether to output variational family with a
mixture of MFVI.
n_mixture: (int) Number of MFVI mixture component to add.
Returns:
q_f, q_sig: (ed.RandomVariable) variational family.
q_f_mean, q_f_sdev: (tf.Variable) variational parameters for q_f
"""
X = tf.convert_to_tensor(X)
Zm = tf.convert_to_tensor(Zm)
Zs = tf.convert_to_tensor(Zs)
Nx, Nm, Ns = X.shape.as_list()[0], Zm.shape.as_list()[0], Zs.shape.as_list(
)[0]
# 1. Prepare constants
# compute matrix constants
Kxx = kern_func(X, ls=ls)
Kmm = kern_func(Zm, ls=ls)
Kxm = kern_func(X, Zm, ls=ls)
Kxs = kern_func(X, Zs, ls=ls)
Kss = kern_func(Zs, ls=ls, ridge_factor=ridge_factor)
# 2. Define variational parameters
# define mean and variance for sigma
q_sig_mean = tf.get_variable(shape=[], name='q_sig_mean')
q_sig_sdev = tf.exp(tf.get_variable(shape=[], name='q_sig_sdev'))
# define free parameters (i.e. mean and full covariance of f_latent)
m = tf.get_variable(shape=[Nm, 1], name='qf_m')
s = tf.get_variable(shape=[Ns * (Ns + 1) / 2], name='qf_s')
L = fill_triangular(s, name='qf_chol')
# components for KL objective
H = tf.eye(Ns) + tf.matmul(L, tf.matmul(Kss, L), transpose_a=True)
cond_cov_inv = tf.matmul(L, tf.matrix_solve(H, tf.transpose(L)))
func_norm_mm = tf.matmul(m, tf.matmul(Kmm, m), transpose_a=True)
log_det_ss = tf.log(tf.matrix_determinant(H))
cond_norm_ss = tf.reduce_sum(tf.multiply(Kss, cond_cov_inv))
# compute sparse gp variational parameter (i.e. mean and covariance of P(f_obs | f_latent))
qf_mean = tf.squeeze(tf.tensordot(Kxm, m, [[1], [0]]), name='qf_mean')
qf_cov = (Kxx -
tf.matmul(Kxs, tf.matmul(cond_cov_inv, Kxs, transpose_b=True)) +
ridge_factor * tf.eye(Nx, dtype=tf.float32))
# define variational family
mixture_par_list = []
if mfvi_mixture:
gp_dist = dist_util.VariationalGaussianProcessDecoupledDistribution(
loc=qf_mean,
covariance_matrix=qf_cov,
func_norm_mm=func_norm_mm,
log_det_ss=log_det_ss,
cond_norm_ss=cond_norm_ss)
q_f, mixture_par_list = inference_util.make_mfvi_sgp_mixture_family(
n_mixture=n_mixture, N=Nx, gp_dist=gp_dist, name='q_f')
else:
q_f = dist_util.VariationalGaussianProcessDecoupled(
loc=qf_mean,
covariance_matrix=qf_cov,
func_norm_mm=func_norm_mm,
log_det_ss=log_det_ss,
cond_norm_ss=cond_norm_ss,
name='q_f')
q_sig = ed.Normal(loc=q_sig_mean, scale=q_sig_sdev, name='q_sig')
return q_f, q_sig, qf_mean, qf_cov, mixture_par_list
def variational_sgpr(X,
Z,
ls=1.,
kern_func=rbf,
ridge_factor=1e-3,
mfvi_mixture=False,
n_mixture=1):
"""Defines the mean-field variational family for GPR.
Args:
X: (np.ndarray of float32) input training features, with dimension (Nx, D).
Z: (np.ndarray of float32) inducing points, with dimension (Nz, D).
ls: (float32) length scale parameter.
kern_func: (function) kernel function.
ridge_factor: (float32) small ridge factor to stabilize Cholesky decomposition
mfvi_mixture: (float32) Whether to output variational family with a
mixture of MFVI.
n_mixture: (int) Number of MFVI mixture component to add.
Returns:
q_f, q_sig: (ed.RandomVariable) variational family.
q_f_mean, q_f_sdev: (tf.Variable) variational parameters for q_f
mixture_par_list: (list of tf.Variable) variational parameters for
MFVI mixture ('mixture_logits', 'mixture_logits_mfvi_mix',
'mean_mfvi', 'sdev_mfvi') if mfvi_mixture=True, else [].
"""
X = tf.convert_to_tensor(X)
Z = tf.convert_to_tensor(Z)
Nx, Nz = X.shape.as_list()[0], Z.shape.as_list()[0]
# 1. Prepare constants
# compute matrix constants
Kxx = kern_func(X, ls=ls)
Kxz = kern_func(X, Z, ls=ls)
Kzz = kern_func(Z, ls=ls, ridge_factor=ridge_factor)
# compute null covariance matrix using Cholesky decomposition
Kzz_chol_inv = tf.matrix_inverse(tf.cholesky(Kzz))
Kzz_inv = tf.matmul(Kzz_chol_inv, Kzz_chol_inv, transpose_a=True)
Kxz_Kzz_chol_inv = tf.matmul(Kxz, Kzz_chol_inv, transpose_b=True)
Kxz_Kzz_inv = tf.matmul(Kxz, Kzz_inv)
Sigma_pre = Kxx - tf.matmul(
Kxz_Kzz_chol_inv, Kxz_Kzz_chol_inv, transpose_b=True)
# 2. Define variational parameters
# define mean and variance for sigma
q_sig_mean = tf.get_variable(shape=[], name='q_sig_mean')
q_sig_sdev = tf.exp(tf.get_variable(shape=[], name='q_sig_sdev'))
# define free parameters (i.e. mean and full covariance of f_latent)
m = tf.get_variable(shape=[Nz], name='qf_m')
s = tf.get_variable(
shape=[Nz * (Nz + 1) / 2],
# initializer=tf.zeros_initializer(),
name='qf_s')
L = fill_triangular(s, name='qf_chol')
S = tf.matmul(L, L, transpose_b=True)
# compute sparse gp variational parameter (i.e. mean and covariance of P(f_obs | f_latent))
qf_mean = tf.tensordot(Kxz_Kzz_inv, m, [[1], [0]], name='qf_mean')
qf_cov = (
Sigma_pre +
tf.matmul(Kxz_Kzz_inv, tf.matmul(S, Kxz_Kzz_inv, transpose_b=True)) +
ridge_factor * tf.eye(Nx, dtype=tf.float32))
# define variational family
mixture_par_list = []
if mfvi_mixture:
gp_dist = tfd.MultivariateNormalFullCovariance(
loc=qf_mean, covariance_matrix=qf_cov)
q_f, mixture_par_list = inference_util.make_mfvi_sgp_mixture_family(
n_mixture=n_mixture, N=Nx, gp_dist=gp_dist, name='q_f')
else:
q_f = ed.MultivariateNormalFullCovariance(loc=qf_mean,
covariance_matrix=qf_cov,
name='q_f')
q_sig = ed.Normal(loc=q_sig_mean, scale=q_sig_sdev, name='q_sig')
return q_f, q_sig, qf_mean, qf_cov, mixture_par_list