def __init__(self, hidden_units:list, input_dim:int, output_dim:int):
"""[summary]
Args:
hidden_units (list): the list of number of RNN unit per stage (for a stacked RNN)
input_dim (int): the number of dimensions one input data point has
output_dim (int): the number of dimensions one output data point has
"""
self.hidden_units = hidden_units
self.input_dim = input_dim
self.output_dim = output_dim
self.weights = weight_variable(shape=[self.hidden_units[-1], output_dim])
self.biases = bias_variable(shape=[output_dim])
self.learning_rate = tf.placeholder(tf.float32)
self.x = tf.placeholder(tf.float32, [None, None, self.input_dim])
self.seqLen = tf.placeholder(tf.int32, [None])
self.y = tf.placeholder(tf.float32, [None, self.output_dim])
self.pred = self._RNN(self.x)
self.cost = self._cost()
self.train_op = tf.train.AdamOptimizer(learning_rate=self.learning_rate).minimize(self.cost)
# Creating the op for initializing all variables
self.init = tf.global_variables_initializer()
# instantiate a global tf session
self.sess = tf.Session()
self.sess.run(self.init)
def VAE(input_shape=[None, 784],
n_components_encoder=2048,
n_components_decoder=2048,
n_hidden=2,
debug=False):
# %%
# Input placeholder
if debug:
input_shape = [50, 784]
x = tf.Variable(np.zeros((input_shape), dtype=np.float32))
else:
x = tf.placeholder(tf.float32, input_shape)
activation = tf.nn.softplus
dims = x.get_shape().as_list()
n_features = dims[1]
W_enc1 = weight_variable([n_features, n_components_encoder])
b_enc1 = bias_variable([n_components_encoder])
h_enc1 = activation(tf.matmul(x, W_enc1) + b_enc1)
W_enc2 = weight_variable([n_components_encoder, n_components_encoder])
b_enc2 = bias_variable([n_components_encoder])
h_enc2 = activation(tf.matmul(h_enc1, W_enc2) + b_enc2)
W_enc3 = weight_variable([n_components_encoder, n_components_encoder])
b_enc3 = bias_variable([n_components_encoder])
h_enc3 = activation(tf.matmul(h_enc2, W_enc3) + b_enc3)
W_mu = weight_variable([n_components_encoder, n_hidden])
b_mu = bias_variable([n_hidden])
W_log_sigma = weight_variable([n_components_encoder, n_hidden])
b_log_sigma = bias_variable([n_hidden])
z_mu = tf.matmul(h_enc3, W_mu) + b_mu
z_log_sigma = 0.5 * (tf.matmul(h_enc3, W_log_sigma) + b_log_sigma)
# %%
# Sample from noise distribution p(eps) ~ N(0, 1)
if debug:
epsilon = tf.random_normal([dims[0], n_hidden])
else:
epsilon = tf.random_normal(tf.stack([tf.shape(x)[0], n_hidden]))
# Sample from posterior
z = z_mu + tf.exp(z_log_sigma) * epsilon
W_dec1 = weight_variable([n_hidden, n_components_decoder])
b_dec1 = bias_variable([n_components_decoder])
h_dec1 = activation(tf.matmul(z, W_dec1) + b_dec1)
W_dec2 = weight_variable([n_components_decoder, n_components_decoder])
b_dec2 = bias_variable([n_components_decoder])
h_dec2 = activation(tf.matmul(h_dec1, W_dec2) + b_dec2)
W_dec3 = weight_variable([n_components_decoder, n_components_decoder])
b_dec3 = bias_variable([n_components_decoder])
h_dec3 = activation(tf.matmul(h_dec2, W_dec3) + b_dec3)
W_mu_dec = weight_variable([n_components_decoder, n_features])
b_mu_dec = bias_variable([n_features])
y = tf.nn.sigmoid(tf.matmul(h_dec3, W_mu_dec) + b_mu_dec)
# p(x|z)
log_px_given_z = -tf.reduce_sum(
x * tf.log(y + 1e-10) + (1 - x) * tf.log(1 - y + 1e-10), 1)
# d_kl(q(z|x)||p(z))
# Appendix B: 0.5 * sum(1 + log(sigma^2) - mu^2 - sigma^2)
kl_div = -0.5 * tf.reduce_sum(
1.0 + 2.0 * z_log_sigma - tf.square(z_mu) - tf.exp(2.0 * z_log_sigma),
1)
loss = tf.reduce_mean(log_px_given_z + kl_div)
return {'cost': loss, 'x': x, 'z': z, 'y': y}
def VAE(input_shape=[None, 784],
n_components_encoder=2048,
n_components_decoder=2048,
n_hidden=2,
debug=False):
# %%
# Input placeholder
if debug:
input_shape = [50, 784]
x = tf.Variable(np.zeros((input_shape), dtype=np.float32))
else:
x = tf.placeholder(tf.float32, input_shape)
activation = tf.nn.softplus
dims = x.get_shape().as_list()
n_features = dims[1]
W_enc1 = weight_variable([n_features, n_components_encoder])
b_enc1 = bias_variable([n_components_encoder])
h_enc1 = activation(tf.matmul(x, W_enc1) + b_enc1)
W_enc2 = weight_variable([n_components_encoder, n_components_encoder])
b_enc2 = bias_variable([n_components_encoder])
h_enc2 = activation(tf.matmul(h_enc1, W_enc2) + b_enc2)
W_enc3 = weight_variable([n_components_encoder, n_components_encoder])
b_enc3 = bias_variable([n_components_encoder])
h_enc3 = activation(tf.matmul(h_enc2, W_enc3) + b_enc3)
W_mu = weight_variable([n_components_encoder, n_hidden])
b_mu = bias_variable([n_hidden])
W_log_sigma = weight_variable([n_components_encoder, n_hidden])
b_log_sigma = bias_variable([n_hidden])
z_mu = tf.matmul(h_enc3, W_mu) + b_mu
z_log_sigma = 0.5 * (tf.matmul(h_enc3, W_log_sigma) + b_log_sigma)
# %%
# Sample from noise distribution p(eps) ~ N(0, 1)
if debug:
epsilon = tf.random_normal(
[dims[0], n_hidden])
else:
epsilon = tf.random_normal(
tf.stack([tf.shape(x)[0], n_hidden]))
# Sample from posterior
z = z_mu + tf.exp(z_log_sigma) * epsilon
W_dec1 = weight_variable([n_hidden, n_components_decoder])
b_dec1 = bias_variable([n_components_decoder])
h_dec1 = activation(tf.matmul(z, W_dec1) + b_dec1)
W_dec2 = weight_variable([n_components_decoder, n_components_decoder])
b_dec2 = bias_variable([n_components_decoder])
h_dec2 = activation(tf.matmul(h_dec1, W_dec2) + b_dec2)
W_dec3 = weight_variable([n_components_decoder, n_components_decoder])
b_dec3 = bias_variable([n_components_decoder])
h_dec3 = activation(tf.matmul(h_dec2, W_dec3) + b_dec3)
W_mu_dec = weight_variable([n_components_decoder, n_features])
b_mu_dec = bias_variable([n_features])
y = tf.nn.sigmoid(tf.matmul(h_dec3, W_mu_dec) + b_mu_dec)
# p(x|z)
log_px_given_z = -tf.reduce_sum(
x * tf.log(y + 1e-10) +
(1 - x) * tf.log(1 - y + 1e-10), 1)
# d_kl(q(z|x)||p(z))
# Appendix B: 0.5 * sum(1 + log(sigma^2) - mu^2 - sigma^2)
kl_div = -0.5 * tf.reduce_sum(
1.0 + 2.0 * z_log_sigma - tf.square(z_mu) - tf.exp(2.0 * z_log_sigma),
1)
loss = tf.reduce_mean(log_px_given_z + kl_div)
return {'cost': loss, 'x': x, 'z': z, 'y': y}
x_d = tf.placeholder(tf.float32, shape=[None, 784])
x_g = tf.placeholder(tf.float32, shape=[None, g_dim])
weights = {
"w_d1": weight_variable([5, 5, 1, 32], "w_d1"),
"w_d2": weight_variable([5, 5, 32, 64], "w_d2"),
"w_d3": weight_variable([7 * 7 * 64, 1], "w_d3"),
"w_g1": weight_variable([g_dim, 4 * 4 * 64], "w_g1"),
"w_g2": weight_variable([5, 5, 32, 64], "w_g2"),
"w_g3": weight_variable([5, 5, 16, 32], "w_g3"),
"w_g4": weight_variable([5, 5, 1, 16], "w_g4")
}
biases = {
"b_d1": bias_variable([32], "b_d1"),
"b_d2": bias_variable([64], "b_d2"),
"b_d3": bias_variable([1], "b_d3"),
"b_g1": bias_variable([4 * 4 * 64], "b_g1"),
"b_g2": bias_variable([32], "b_g2"),
"b_g3": bias_variable([16], "b_g3"),
"b_g4": bias_variable([1], "b_g4"),
}
var_d = [
weights["w_d1"], weights["w_d2"], weights["w_d3"], biases["b_d1"],
biases["b_d2"], biases["b_d3"]
]
var_g = [
weights["w_g1"], weights["w_g2"], weights["w_g3"], weights["w_g4"],
biases["b_g1"], biases["b_g2"], biases["b_g3"], biases["b_g4"]
def VAE_model(x, n_components_encoder=2048, n_components_decoder=2048, n_hidden=2):
activation = tf.nn.softplus
dims = x.get_shape().as_list()
n_features = dims[1]
W_enc1 = weight_variable([n_features, n_components_encoder])
b_enc1 = bias_variable([n_components_encoder])
h_enc1 = activation(tf.matmul(x, W_enc1) + b_enc1)
W_enc2 = weight_variable([n_components_encoder, n_components_encoder])
b_enc2 = bias_variable([n_components_encoder])
h_enc2 = activation(tf.matmul(h_enc1, W_enc2) + b_enc2)
W_enc3 = weight_variable([n_components_encoder, n_components_encoder])
b_enc3 = bias_variable([n_components_encoder])
h_enc3 = activation(tf.matmul(h_enc2, W_enc3) + b_enc3)
W_mu = weight_variable([n_components_encoder, n_hidden])
b_mu = bias_variable([n_hidden])
W_log_sigma = weight_variable([n_components_encoder, n_hidden])
b_log_sigma = bias_variable([n_hidden])
z_mu = tf.matmul(h_enc3, W_mu) + b_mu
z_log_sigma = 0.5 * (tf.matmul(h_enc3, W_log_sigma) + b_log_sigma)
# %%
# Sample from noise distribution p(eps) ~ N(0, 1)
epsilon = tf.random_normal(tf.stack([tf.shape(x)[0], n_hidden]))
# Sample from posterior
z = z_mu + tf.exp(z_log_sigma) * epsilon
W_dec1 = weight_variable([n_hidden, n_components_decoder])
b_dec1 = bias_variable([n_components_decoder])
h_dec1 = activation(tf.matmul(z, W_dec1) + b_dec1)
W_dec2 = weight_variable([n_components_decoder, n_components_decoder])
b_dec2 = bias_variable([n_components_decoder])
h_dec2 = activation(tf.matmul(h_dec1, W_dec2) + b_dec2)
W_dec3 = weight_variable([n_components_decoder, n_components_decoder])
b_dec3 = bias_variable([n_components_decoder])
h_dec3 = activation(tf.matmul(h_dec2, W_dec3) + b_dec3)
W_mu_dec = weight_variable([n_components_decoder, n_features])
b_mu_dec = bias_variable([n_features])
y = tf.nn.sigmoid(tf.matmul(h_dec3, W_mu_dec) + b_mu_dec)
# p(x|z)
log_px_given_z = -tf.reduce_sum(
x * tf.log(y + 1e-10) +
(1 - x) * tf.log(1 - y + 1e-10), 1)
# d_kl(q(z|x)||p(z))
# Appendix B: 0.5 * sum(1 + log(sigma^2) - mu^2 - sigma^2)
kl_div = -0.5 * tf.reduce_sum(
1.0 + 2.0 * z_log_sigma - tf.square(z_mu) - tf.exp(2.0 * z_log_sigma),
1)
loss = tf.reduce_mean(log_px_given_z + kl_div)
learning_rate = 0.001
optimizer = tf.train.AdamOptimizer(learning_rate).minimize(loss)
return [loss, optimizer, y, z]
