def sample_structure(self, mu, v):
if self.sigma_type == 'const':
vv = self.sigma * torch.ones(mu.size())
else:
vv = v
sample = sample_gaussian(mu, vv)
return sample
def compute_nelbo(self, x):
# encode x and draw sample
mu_z, var_z = self.enc(x)
z = ut.sample_gaussian(mu_z, var_z)
mu_x, var_x = self.dec(z)
dkl = ut.kl_normal(mu_z, var_z, self.prior_mu, self.prior_var)
recon = -ut.log_normal(x, mu_x, var_x)
nelbo = dkl + recon
return nelbo.mean(), dkl.mean(), recon.mean()
def _initialize_vars(self):
hidden_p = tf.nn.sigmoid(tf.matmul(self.x, self.w) + self.hidden_bias)
visible_recon_p = tf.matmul(sample_bernoulli(hidden_p),
tf.transpose(self.w)) + self.visible_bias
if self.sample_visible:
visible_recon_p = sample_gaussian(visible_recon_p, self.sigma)
hidden_recon_p = tf.nn.sigmoid(
tf.matmul(visible_recon_p, self.w) + self.hidden_bias)
positive_grad = tf.matmul(tf.transpose(self.x), hidden_p)
negative_grad = tf.matmul(tf.transpose(visible_recon_p),
hidden_recon_p)
def f(x_old, x_new):
return self.momentum * x_old +\
self.learning_rate * x_new * (1 - self.momentum) / tf.to_float(tf.shape(x_new)[0])
delta_w_new = f(self.delta_w, positive_grad - negative_grad)
delta_visible_bias_new = f(self.delta_visible_bias,
tf.reduce_mean(self.x - visible_recon_p, 0))
delta_hidden_bias_new = f(self.delta_hidden_bias,
tf.reduce_mean(hidden_p - hidden_recon_p, 0))
update_delta_w = self.delta_w.assign(delta_w_new)
update_delta_visible_bias = self.delta_visible_bias.assign(
delta_visible_bias_new)
update_delta_hidden_bias = self.delta_hidden_bias.assign(
delta_hidden_bias_new)
update_w = self.w.assign(self.w + delta_w_new)
update_visible_bias = self.visible_bias.assign(self.visible_bias +
delta_visible_bias_new)
update_hidden_bias = self.hidden_bias.assign(self.hidden_bias +
delta_hidden_bias_new)
self.update_deltas = [
update_delta_w, update_delta_visible_bias, update_delta_hidden_bias
]
self.update_weights = [
update_w, update_visible_bias, update_hidden_bias
]
self.compute_hidden = tf.nn.sigmoid(
tf.matmul(self.x, self.w) + self.hidden_bias)
self.compute_visible = tf.matmul(
self.compute_hidden, tf.transpose(self.w)) + self.visible_bias
self.compute_visible_from_hidden = tf.matmul(
self.y, tf.transpose(self.w)) + self.visible_bias
def sample_x_from_z(self, z):
mu, var = self.dec(z)
return ut.sample_gaussian(mu, var), mu
def sample_z(self, num_samples):
return ut.sample_gaussian(self.prior_mu.expand(num_samples, -1),
self.prior_var.expand(num_samples, -1))
def sample_z_from_x(self, x):
return ut.sample_gaussian(*self.enc(x))
def _initialize_vars(self):
self.dropout_rate = tf.placeholder(tf.float32,
shape=(),
name="dropout_rate") ###dropout
hidden_mf = tf.matmul(self.x, self.w) + self.hidden_bias
hidden_p = tf.nn.sigmoid(hidden_mf)
hidden_act = self.sample_relu(hidden_mf, hidden_p)
# dropout_cte = tf.nn.dropout(tf.ones_like(hidden_mf), self.dropout_rate)###dropout
# hidden_act = tf.multiply(hidden_act, dropout_cte)###dropout
visible_recon_p = tf.matmul(hidden_act, tf.transpose(
self.w)) + self.visible_bias
visible_recon_act = sample_gaussian(visible_recon_p, self.sigma)
hidden_recon_mf = tf.matmul(visible_recon_act,
self.w) + self.hidden_bias
hidden_recon_p = tf.nn.sigmoid(hidden_recon_mf)
hidden_recon_act = self.sample_relu(hidden_recon_mf, hidden_recon_p)
positive_grad = tf.matmul(tf.transpose(
self.x), hidden_act) / tf.to_float(tf.shape(self.x)[0])
# negative_grad = tf.matmul(tf.transpose(visible_recon_act), hidden_recon_act)/tf.to_float(tf.shape(self.x)[0])
negative_grad = tf.matmul(tf.transpose(visible_recon_p),
hidden_recon_act) / tf.to_float(
tf.shape(self.x)[0])
def f(x_old, x_new):
return self.momentum * x_old +\
self.learning_rate_ph * x_new * (1 - self.momentum)
delta_w_new = f(
self.delta_w, positive_grad -
negative_grad) - self.learning_rate_ph * self.l1 * tf.abs(self.w)
# delta_visible_bias_new = f(self.delta_visible_bias, tf.reduce_mean(self.x - visible_recon_act, 0))/128 #+ self.learning_rate_ph*self.l1*tf.abs(self.visible_bias)
# delta_hidden_bias_new = f(self.delta_hidden_bias, tf.reduce_mean(hidden_p - hidden_recon_p, 0))/128 #+ self.learning_rate_ph*self.l1*tf.abs(self.hidden_bias)
# delta_visible_bias_new = f(self.delta_visible_bias, tf.reduce_mean(self.x - visible_recon_act, 0))#/128 #+ self.learning_rate_ph*self.l1*tf.abs(self.visible_bias)
delta_visible_bias_new = f(
self.delta_visible_bias, tf.reduce_mean(
self.x - visible_recon_p,
0)) #+ self.learning_rate_ph*self.l1*tf.abs(self.visible_bias)
delta_hidden_bias_new = f(
self.delta_hidden_bias,
tf.reduce_mean(
hidden_act - hidden_recon_act,
0)) # + self.learning_rate_ph*self.l1*tf.abs(self.hidden_bias)
#hidden_act
update_delta_w = self.delta_w.assign(delta_w_new)
update_delta_visible_bias = self.delta_visible_bias.assign(
delta_visible_bias_new)
update_delta_hidden_bias = self.delta_hidden_bias.assign(
delta_hidden_bias_new)
update_w = self.w.assign(self.w + delta_w_new)
update_visible_bias = self.visible_bias.assign(self.visible_bias +
delta_visible_bias_new)
update_hidden_bias = self.hidden_bias.assign(self.hidden_bias +
delta_hidden_bias_new)
self.update_deltas = [
update_delta_w, update_delta_visible_bias, update_delta_hidden_bias
]
with tf.control_dependencies(self.update_deltas):
self.update_weights = [
update_w, update_visible_bias, update_hidden_bias
]
if self.train_sigma:
delta_sigma_new = 0.01 * f(
self.delta_sigma,
tf.reduce_mean(
(self.x**2 - visible_recon_p**2 + 2 * self.visible_bias *
(visible_recon_p - self.x)) / (self.sigma), 0, False))
update_delta_sigma = self.delta_sigma.assign(delta_sigma_new)
update_sigma = self.sigma.assign(self.sigma + delta_sigma_new)
self.update_deltas += [update_delta_sigma]
self.update_weights += [update_sigma]
self.compute_hidden = tf.nn.relu(
tf.matmul(self.x, self.w) + self.hidden_bias)
self.compute_visible = tf.matmul(
self.compute_hidden, tf.transpose(self.w)) + self.visible_bias
self.compute_visible_from_hidden = tf.matmul(
self.y, tf.transpose(self.w)) + self.visible_bias
self.features = self.compute_hidden
self.featuresv2 = tf.nn.relu(tf.matmul(self.x, self.w))
self.featuresv3 = tf.nn.sigmoid(
tf.matmul(self.x, self.w) + self.hidden_bias)
self.featuresv4 = tf.matmul(self.x, self.w)
# self.fix_hidden = self.hidden_bias.assign(0.1*(-1/2) + 0.1*1*(self.hidden_bias - tf.reduce_min(self.hidden_bias))/(tf.reduce_max(self.hidden_bias)-tf.reduce_min(self.hidden_bias)))
self.fix_hidden = self.hidden_bias.assign(-0.05 + 0.1 * (
(-1 / 2) + (self.hidden_bias - tf.reduce_min(self.hidden_bias)) /
(tf.reduce_max(self.hidden_bias) - tf.reduce_min(self.hidden_bias))
))
self.restart_hidden = self.hidden_bias.assign(
0.5 * tf.ones_like(self.hidden_bias))