def bayes_dense_2(x, num_units, name='dense', gamma=1.0, activation=None):
with tf.variable_scope(name, reuse=tf.AUTO_REUSE):
W_mu = tf.get_variable('W_mu', [x.shape[1], num_units])
W_rho = tf.nn.softplus(
tf.get_variable('W_rho', [x.shape[1], num_units],
initializer=tf.random_uniform_initializer(-3., -2.)))
b_mu = tf.get_variable('b_mu', [num_units],
initializer=tf.zeros_initializer())
b_rho = tf.nn.softplus(
tf.get_variable('b_rho', [num_units],
initializer=tf.random_uniform_initializer(-3., -2.)))
xW_mean = tf.matmul(x, W_mu)
xW_std = tf.sqrt(tf.matmul(tf.square(x), tf.square(W_rho)) + 1e-6)
xW = xW_mean + xW_std*tf.random.normal(tf.shape(xW_mean))
b = b_mu + b_rho * tf.random.normal(b_mu.shape)
x = xW + b
if activation == 'relu':
x = tf.nn.relu(x)
# kl divergence
kld_W = tf.reduce_sum(kl_divergence(Normal(W_mu, W_rho), Normal(0., gamma)))
kld_b = tf.reduce_sum(kl_divergence(Normal(b_mu, b_rho), Normal(0., gamma)))
kld = kld_W + kld_b
return x, kld
def Gelu(self, x):
'''Implementation for GELU Activation Function
Arguments:
x (tensor):
Input tensor
Returns:
Tensor, output of 'GELU' activation function
'''
normal = Normal(loc=0., scale=1.)
return x * normal.cdf(x)
def _build_anet(self, name, trainable):
with tf.compat.v1.variable_scope(name):
l1 = tf.compat.v1.layers.dense(self.tfs, 200, tf.nn.relu, trainable=trainable)
mu = A_BOUND * tf.compat.v1.layers.dense(l1, A_DIM, tf.nn.tanh, trainable=trainable)
sigma = tf.compat.v1.layers.dense(l1, A_DIM, tf.nn.softplus, trainable=trainable)
norm_dist = Normal(loc=mu, scale=sigma)
params = tf.compat.v1.get_collection(tf.compat.v1.GraphKeys.GLOBAL_VARIABLES, scope=name)
return norm_dist, params
def gaussian_logp(y, logd):
assert y.shape[0] == logd.shape[
0] and y.dtype == logd.dtype and logd.shape.ndims == 1
orig_dtype = y.dtype
y, logd = map(tf.to_float, [y, logd])
return tf.cast(
tf.reduce_sum(Normal(0., 1.).log_prob(y)) / int(y.shape[0]) +
tf.reduce_mean(logd),
dtype=orig_dtype)
def build_forward(*, x, dequant_flow, flow, flow_kwargs):
dequant_x, dequant_logd = dequant_flow.forward(x, **flow_kwargs)
y, main_logd = flow.forward(dequant_x, **flow_kwargs)
logp = sumflat(Normal(0., 1.).log_prob(y))
assert dequant_logd.shape == main_logd.shape == logp.shape == [
y.shape[0]
] == [dequant_x.shape[0]] == [x.shape[0]]
total_logp = dequant_logd + main_logd + logp
loss = -tf.reduce_mean(total_logp)
return dequant_x, y, loss, total_logp
def inverse(self, y, flow_kwargs):
'''
g = f^{-1}
y = g^{-1}(xz)
p(y) = p(xz) * |det(J_g^{-1}^{-1}(y))| = p(xz) * |det(J_g(y))|
'''
y = tf.reshape(y, [-1] + y.shape.as_list()[-3:])
logpy = sumflat(Normal(0., 1.).log_prob(y))
xz = y
xz = tf.reshape(xz, [-1, 8, 8, 64*(x_dims+extra_dims)])
xz, logd_flow = self.flow.inverse(xz, **flow_kwargs)
return xz, logpy + logd_flow
def forward(self, x, z, flow_kwargs):
'''
y = f(xz)
xz = f^{-1}(y)
p(xz) = p(y) * |det(J_f^{-1}^{-1}(xz))| = p(y) * |det(J_f(xz))|
'''
H, W, Cx = x.shape.as_list()[-3:]
x = tf.reshape(x, [-1, H, W, Cx])
z = tf.reshape(z, [-1, H, W, extra_dims])
xz = tf.concat([x, z], axis=-1)
y, logd_flow = self.flow.forward(xz, **flow_kwargs)
logpy = sumflat(Normal(0., 1.).log_prob(y))
return y, logpy + logd_flow
def __init__(self, n_input, n_list, n_y=None,y_weight=100):
'''
n_input - number of input neurons
n_list - list of numbers of neurons in the hidden layers
n_y: optional - number of features that will be given as input y during training
y_weight - relative weight of losses (VAE vs regression for y features). Trial-and-error
'''
# input data
self.X = tf.placeholder(tf.float32, shape=(None, n_input))
# input y features
if (n_y is not None):
self.y = tf.placeholder(tf.float32, shape=(None, n_y))
# encoder
self.encoder_layers = []
# input layer
previous = n_input
# current is the output of each layer (skip last because there is nothing after it)
for current in n_list[:-1]:
h = DenseLayer(previous,current)
self.encoder_layers.append(h)
previous = current
# latent features number
latent = n_list[-1]
encoder_output = DenseLayer(current,latent*2,activation='none')
self.encoder_layers.append(encoder_output)
# feed forward through encoder
c_X = self.X
for layer in self.encoder_layers:
c_X = layer.feed_forward(c_X)
# c_X now holds the output of the encoder
# first half are the means
self.means = c_X[:,:latent]
# second half are the std; must be positive; +1e-6 for smoothing
self.std = tf.nn.softplus(c_X[:,latent:]) + 1e-6
# optional loss for steered latent features
if (n_y is not None):
self.yhat = self.means[:,:n_y]
self.error = tf.losses.mean_squared_error(labels=self.y,predictions=self.yhat)
# reparameterization trick
normal = Normal(loc=self.means,scale=self.std)
self.Z = normal.sample()
# decoder
self.decoder_layers = []
previous = latent
for current in reversed(n_list[:-1]):
h = DenseLayer(previous,current)
self.decoder_layers.append(h)
previous = current
# output is the reconstruction
decoder_output = DenseLayer(previous,n_input,activation=lambda x:x)
self.decoder_layers.append(decoder_output)
#feed forward through decoder, using the sampled 'data'
c_X = self.Z
for layer in self.decoder_layers:
c_X = layer.feed_forward(c_X)
logits = c_X
# use logits for cost function below
neg_cross_entropy = -tf.nn.sigmoid_cross_entropy_with_logits(labels=self.X,
logits=logits)
neg_cross_entropy = tf.reduce_sum(neg_cross_entropy, 1)
# output
self.y_prob = Bernoulli(logits=logits)
# sample from output
self.post_pred = self.y_prob.sample()
self.post_pred_probs = tf.nn.sigmoid(logits)
# generate 'de-novo' output
self.gen = tf.Variable(0)
Z_std = Normal(0.0,1.0).sample([self.gen,latent])
c_X = Z_std
for layer in self.decoder_layers:
c_X = layer.feed_forward(c_X)
logits = c_X
prior_pred_dist = Bernoulli(logits=logits)
self.prior_pred = prior_pred_dist.sample()
self.prior_pred_probs = tf.nn.sigmoid(logits)
# manually input Z
self.Z_input = tf.placeholder(np.float32, shape=(None, latent))
c_X = self.Z_input
for layer in self.decoder_layers:
c_X = layer.feed_forward(c_X)
logits = c_X
self.manual_prior_prob = tf.nn.sigmoid(logits)
# cost function
# Kullback–Leibler divergence
kl = -tf.log(self.std) + 0.5*(self.std**2 + self.means**2) - 0.5
kl = tf.reduce_sum(kl, axis=1)
# ELBO
self.elbo = tf.reduce_sum(neg_cross_entropy - kl)
if (n_y is None):
# only ELBO
self.optimizer = tf.train.RMSPropOptimizer(learning_rate=0.001).minimize(-self.elbo)
else:
# weighted regression loss and ELBO
self.optimizer = tf.train.RMSPropOptimizer(learning_rate=0.001).minimize(
tf.reduce_sum(y_weight*self.error-self.elbo))
self.init = tf.global_variables_initializer()
self.session = tf.Session()
self.session.run(self.init)
y_train_np = func(x_train_np, 0.03)
x_test_np = np.random.rand(num_test, 1)
y_test_np = func(x_test_np, 0.03)
np.random.seed(int(time.time()))
x = tf.placeholder(tf.float32, shape=[None, 1])
y = tf.placeholder(tf.float32, shape=[None, 1])
out, kld1 = bayes_dense(x, 100, activation='relu', name='dense1')
out, kld2 = bayes_dense(out, 100, activation='relu', name='dense2')
out, kld3 = bayes_dense(out, 2, name='dense3')
mu, sigma = tf.split(out, 2, axis=-1)
sigma = tf.nn.softplus(sigma)
# log-likelihood
ll = tf.reduce_mean(Normal(mu, sigma).log_prob(y))
# kl-divergence
kld = args.kl_coeff * (kld1 + kld2 + kld3) / np.float32(num_train)
if not args.test:
if not os.path.isdir('results/bnn'):
os.makedirs('results/bnn')
saver = tf.train.Saver(tf.trainable_variables())
lr = tf.placeholder(tf.float32)
train_op = tf.train.AdamOptimizer(lr).minimize(-ll + kld)
def get_lr(t):
if t < 0.25 * args.num_steps:
return args.lr
elif t < 0.5 * args.num_steps:
def __init__(self, n_input, n_list, ent_weight=1, lr=0.001):
'''
number of input neurons and a list of the number of neurons in the encoder hidden layers
The last number provided should be the number of latent features desired
The decoder will have an inverted architecture
Note: the actual number of neurons in the last layer of the encoder will be x2, for mean and std
'''
# input data
self.X = tf.placeholder(tf.float32, shape=(None, n_input))
# encoder
self.encoder_layers = []
# input layer
previous = n_input
# in case there is only one hidden layer (for loop will be skipped)
current = n_input
# current is the output of each layer (skip last because there is nothing after it)
for current in n_list[:-1]:
h = DenseLayer(previous, current)
self.encoder_layers.append(h)
previous = current
# latent features number
latent = n_list[-1]
encoder_output = DenseLayer(current, latent * 2, activation='none')
self.encoder_layers.append(encoder_output)
# feed forward through encoder
c_X = self.X
for layer in self.encoder_layers:
c_X = layer.feed_forward(c_X)
# c_X now holds the output of the encoder
# first half are the means
self.means = c_X[:, :latent]
# second half are the std; must be positive; +1e-6 for smoothing
self.std = tf.nn.softplus(c_X[:, latent:]) + 1e-6
# reparameterization trick
normal = Normal(loc=self.means, scale=self.std)
self.Z = normal.sample()
# decoder
self.decoder_layers = []
previous = latent
for current in reversed(n_list[:-1]):
h = DenseLayer(previous, current)
self.decoder_layers.append(h)
previous = current
# output is the reconstruction
decoder_output = DenseLayer(previous, n_input, activation=lambda x: x)
self.decoder_layers.append(decoder_output)
#feed forward through decoder, using the sampled 'data'
c_X = self.Z
for layer in self.decoder_layers:
c_X = layer.feed_forward(c_X)
logits = c_X
# use logits for cost function below
neg_cross_entropy = -tf.nn.sigmoid_cross_entropy_with_logits(
labels=self.X, logits=logits)
neg_cross_entropy = tf.reduce_sum(neg_cross_entropy, 1)
# output
self.y_prob = Bernoulli(logits=logits)
# sample from output
self.post_pred = self.y_prob.sample()
self.post_pred_probs = tf.nn.sigmoid(logits)
# generate 'de-novo' output
self.gen = tf.Variable(0)
Z_std = Normal(0.0, 1.0).sample([self.gen, latent])
c_X = Z_std
for layer in self.decoder_layers:
c_X = layer.feed_forward(c_X)
logits = c_X
prior_pred_dist = Bernoulli(logits=logits)
self.prior_pred = prior_pred_dist.sample()
self.prior_pred_probs = tf.nn.sigmoid(logits)
# manually input Z
self.Z_input = tf.placeholder(np.float32, shape=(None, latent))
c_X = self.Z_input
for layer in self.decoder_layers:
c_X = layer.feed_forward(c_X)
logits = c_X
self.manual_prior_prob = tf.nn.sigmoid(logits)
# cost function
# Kullback–Leibler divergence
kl = -tf.log(self.std) + 0.5 * (self.std**2 + self.means**2) - 0.5
kl = tf.reduce_sum(kl, axis=1)
# ELBO
self.elbo = tf.reduce_sum(ent_weight * neg_cross_entropy - kl)
self.optimizer = tf.train.RMSPropOptimizer(
learning_rate=lr).minimize(-self.elbo)
self.init = tf.global_variables_initializer()
self.session = tf.Session()
self.session.run(self.init)
def gaussian_sample_logp(shape, dtype):
eps = tf.random_normal(shape)
logp = Normal(0., 1.).log_prob(eps)
assert logp.shape == eps.shape
logp = tf.reduce_sum(tf.layers.flatten(logp), axis=1)
return tf.cast(eps, dtype=dtype), tf.cast(logp, dtype=dtype)
def prob(args):
# mean, std, act_taken = args
mean, act_taken = args
std = np.float64(0.75)
prob = Normal(loc=mean, scale=std).prob(act_taken)
return prob
def __init__(self,
image_shape=(128, 128, 3),
conv_param=(3, 16, True),
n_list=[256, 32]):
# input data
self.X = tf.placeholder(tf.float32, shape=(None, *image_shape))
# encoder
self.encoder_layers = []
# convolution layer
h = Conv2DLayer(image_shape[2], conv_param[0], conv_param[1],
conv_param[2])
self.encoder_layers.append(h)
# flatten layer
self.encoder_layers.append(FlattenLayer())
# calculate number of input neurons to the FC layer
previous = image_shape[0] * image_shape[1] * conv_param[1]
if conv_param[2]:
previous = previous // 4
# save for later
flat = previous
# current is the output of each layer (skip last because there is nothing after it)
for current in n_list[:-1]:
h = DenseLayer(previous, current)
self.encoder_layers.append(h)
previous = current
# latent features number
latent = n_list[-1]
encoder_output = DenseLayer(current, latent * 2, activation='none')
self.encoder_layers.append(encoder_output)
# feed forward through encoder
c_X = self.X
for layer in self.encoder_layers:
c_X = layer.feed_forward(c_X)
# c_X now holds the output of the encoder
# first half are the means
self.means = c_X[:, :latent]
# second half are the std; must be positive; +1e-6 for smoothing
self.std = tf.nn.softplus(c_X[:, latent:]) + 1e-6
# reparameterization trick
normal = Normal(loc=self.means, scale=self.std)
self.Z = normal.sample()
# decoder
self.decoder_layers = []
previous = latent
for current in reversed(n_list[:-1]):
h = DenseLayer(previous, current)
self.decoder_layers.append(h)
previous = current
decoder_output = DenseLayer(previous, flat, activation=lambda x: x)
self.decoder_layers.append(decoder_output)
#feed forward through decoder, using the sampled 'data'
c_X = self.Z
for layer in self.decoder_layers:
c_X = layer.feed_forward(c_X)
# reshape
if (conv_param[2]):
shape = [
-1, image_shape[0] // 2, image_shape[0] // 2, conv_param[1]
]
else:
shape = [-1, image_shape[0], image_shape[0], conv_param[1]]
c_X = tf.reshape(c_X, shape)
# convolution transpose
self.trans_k = tf.Variable(
tf.truncated_normal(
[conv_param[0], conv_param[0], image_shape[2], conv_param[1]],
stddev=0.1))
if (conv_param[2]):
strides = (1, 2, 2, 1)
else:
strides = (1, 1, 1, 1)
c_X = tf.nn.conv2d_transpose(c_X,
self.trans_k,
strides=strides,
padding='SAME',
output_shape=[50, *image_shape])
# output logit
logits = c_X
# use logits for cost function below
neg_cross_entropy = -tf.nn.sigmoid_cross_entropy_with_logits(
labels=self.X, logits=logits)
neg_cross_entropy = tf.reduce_sum(neg_cross_entropy, 1)
# output
self.y_prob = Bernoulli(logits=logits)
# sample from output
self.post_pred = self.y_prob.sample()
self.post_pred_probs = tf.nn.sigmoid(logits)
# generate 'de-novo' output
self.gen = tf.Variable(0)
Z_std = Normal(0.0, 1.0).sample([self.gen, latent])
c_X = Z_std
for layer in self.decoder_layers:
c_X = layer.feed_forward(c_X)
c_X = tf.reshape(c_X, shape)
c_X = tf.nn.conv2d_transpose(c_X,
self.trans_k,
strides=strides,
padding='SAME',
output_shape=[50, *image_shape])
logits = c_X
prior_pred_dist = Bernoulli(logits=logits)
self.prior_pred = prior_pred_dist.sample()
self.prior_pred_probs = tf.nn.sigmoid(logits)
# manually input Z
self.Z_input = tf.placeholder(np.float32, shape=(None, latent))
c_X = self.Z_input
for layer in self.decoder_layers:
c_X = layer.feed_forward(c_X)
logits = c_X
self.manual_prior_prob = tf.nn.sigmoid(logits)
# cost function
# Kullback–Leibler divergence
kl = -tf.log(self.std) + 0.5 * (self.std**2 + self.means**2) - 0.5
kl = tf.reduce_sum(kl, axis=1)
# ELBO
self.elbo = tf.reduce_sum(neg_cross_entropy - kl)
self.optimizer = tf.train.RMSPropOptimizer(
learning_rate=0.001).minimize(-self.elbo)
self.init = tf.global_variables_initializer()
self.session = tf.Session()
self.session.run(self.init)
def log_prob(args):
# mean, std, act_taken = args
mean, act_taken = args
std = 0.75
log_prob = Normal(loc=mean, scale=std).log_prob(act_taken)
return log_prob
def gps_action(args):
# mean, std = args
mean = args
std = np.float64(0.75)
action = Normal(loc=mean[0], scale=std).sample(1)
return [action[0], mean[0]]
def action(args):
# mean, std = args
mean = args
std = 0.75
action = Normal(loc=mean[0], scale=std).sample(1)
return [action[0], mean[0]]
def probit(x):
normal = TFNormal(loc=0., scale=1.)
return normal.cdf(x)
x = tf.placeholder(tf.float32, shape=[None, 1])
y = tf.placeholder(tf.float32, shape=[None, 1])
# define network
# BayesDense(1, 100) - ReLU
# BayesDense(100, 100) - ReLU
# BayesDense(100, 2)
# make sure to collect every kl-divergences
#########################################################
out, kld1 =
#########################################################
mu, sigma = tf.split(out, 2, axis=-1)
sigma = tf.nn.softplus(sigma)
# log-likelihood
ll = tf.reduce_mean(Normal(mu, sigma).log_prob(y))
# kl-divergence
kld = args.kl_coeff * (kld1 + kld2 + kld3) / np.float32(num_train)
if not args.test:
if not os.path.isdir('results/bnn'):
os.makedirs('results/bnn')
saver = tf.train.Saver(tf.trainable_variables())
lr = tf.placeholder(tf.float32)
#########################################################
train_op = tf.train.AdamOptimizer(lr).minimize(########)
#########################################################
def get_lr(t):
if t < 0.25 * args.num_steps:
class OutputNodeClassification(OutputNodeBase):
def __init__(self, y_train_tf, y_test_tf, n_samples, variance_bias, dtype):
OutputNodeBase.__init__(self, y_train_tf, y_test_tf, n_samples)
self.norm = Normal(loc=tf.constant(0.0, dtype),
scale=tf.constant(1.0, dtype))
self.dtype = dtype
self.variance_bias = variance_bias
def set_nodes_as_input(self, list_of_input_nodes):
# Override from BaseNode
assert (
len(list_of_input_nodes) == 1
), "The last node should have just one input, currently {}".format(
len(list_of_input_nodes))
self.list_of_input_nodes = list_of_input_nodes
def get_logz(self):
# Using step function as likelihood with noise is equivalent to Gaussian cdf.
# Calculate:
# log E[ p(y | f^L)] = log \int p(y | f^L) q^\cavity(f^L | x)
# = log \int p(y | f^L) N(f^L | input_mean, input_vars)
# with p(y | f^L) a step function (f^L bigger or equal to 0 classified as y=1)
# By samples:
# log 1/S \sum_{s=1}^S \Phi((y_{i,s} * input_means) / \sqrt(input_vars))
# Using log cdf for robustness
input_means, input_vars = self.get_input()
S = tf.shape(input_means)[0]
# Parameter of the log cdf
alpha = (tf.cast(self.y_train_tf, self.dtype) * input_means /
tf.sqrt(input_vars + self.variance_bias))
return tf.reduce_logsumexp(self.norm.log_cdf(alpha), 0) - tf.log(
tf.cast(S, self.dtype))
def calculate_log_likelihood(self):
# The only difference with the function above is that
# the means and vars should be calculated using the psoterior instead of the cavity
# and y_test should be used.
input_means, input_vars = self.get_input()
S = tf.shape(input_means)[0]
# Parameter of the log cdf
alpha = (tf.cast(self.y_test_tf, self.dtype) * input_means /
tf.sqrt(input_vars + self.variance_bias))
return tf.reduce_logsumexp(self.norm.log_cdf(alpha), 0) - tf.log(
tf.cast(S, self.dtype))
def get_predicted_values(self):
input_means, input_vars = self.get_input() # S, N, 1
S = tf.shape(input_means)[0]
# (S, N, 1)
alpha = input_means / tf.sqrt(input_vars + self.variance_bias)
# (N, 1)
prob = tf.exp(
tf.reduce_logsumexp(self.norm.log_cdf(alpha), 0) -
tf.log(tf.cast(S, self.dtype)))
# label[n] = -1 if input_means[n] < 0 else 1
labels = tf.where(
tf.less(tf.reduce_sum(input_means, 0), tf.zeros_like(prob)),
-1 * tf.ones_like(prob),
tf.ones_like(prob),
)
return labels, prob
def sample_from_latent(self):
input_means, input_vars = self.get_input() # S, N, 1
# Returns samples from H^L
return tf.random_normal(
tf.shape(self.input_means),
mean=self.input_means,
stddev=tf.sqrt(self.input_vars),
seed=3,
dtype=self.dtype,
) # seed=3
def __init__(self, y_train_tf, y_test_tf, n_samples, variance_bias, dtype):
OutputNodeBase.__init__(self, y_train_tf, y_test_tf, n_samples)
self.norm = Normal(loc=tf.constant(0.0, dtype),
scale=tf.constant(1.0, dtype))
self.dtype = dtype
self.variance_bias = variance_bias