def _buildGraph(self):
x_in = tf.placeholder(tf.float32, shape=[None, # enables variable batch size
self.architecture[0]], name="x")
dropout = tf.placeholder_with_default(1., shape=[], name="dropout")
# encoding / "recognition": q(z|x)
encoding = [Dense("encoding", hidden_size, dropout, self.nonlinearity)
# hidden layers reversed for function composition: outer -> inner
for hidden_size in reversed(self.architecture[1:-1])]
h_encoded = composeAll(encoding)(x_in)
# latent distribution parameterized by hidden encoding
# z ~ N(z_mean, np.exp(z_log_sigma)**2)
z_mean = Dense("z_mean", self.architecture[-1], dropout)(h_encoded)
z_log_sigma = Dense("z_log_sigma", self.architecture[-1], dropout)(h_encoded)
# kingma & welling: only 1 draw necessary as long as minibatch large enough (>100)
z = self.sampleGaussian(z_mean, z_log_sigma)
# decoding / "generative": p(x|z)
decoding = [Dense("decoding", hidden_size, dropout, self.nonlinearity)
for hidden_size in self.architecture[1:-1]] # assumes symmetry
# final reconstruction: restore original dims, squash outputs [0, 1]
decoding.insert(0, Dense( # prepend as outermost function
"x_decoding", self.architecture[0], dropout, self.squashing))
x_reconstructed = tf.identity(composeAll(decoding)(z), name="x_reconstructed")
# reconstruction loss: mismatch b/w x & x_reconstructed
# binary cross-entropy -- assumes x & p(x|z) are iid Bernoullis
rec_loss = VAE.crossEntropy(x_reconstructed, x_in)
# Kullback-Leibler divergence: mismatch b/w approximate vs. imposed/true posterior
kl_loss = VAE.kullbackLeibler(z_mean, z_log_sigma)
with tf.name_scope("l2_regularization"):
regularizers = [tf.nn.l2_loss(var) for var in self.sess.graph.get_collection(
"trainable_variables") if "weights" in var.name]
l2_reg = self.lambda_l2_reg * tf.add_n(regularizers)
with tf.name_scope("cost"):
# average over minibatch
cost = tf.reduce_mean(rec_loss + kl_loss, name="vae_cost")
cost += l2_reg
# optimization
global_step = tf.Variable(0, trainable=False)
with tf.name_scope("Adam_optimizer"):
optimizer = tf.train.AdamOptimizer(self.learning_rate)
tvars = tf.trainable_variables()
grads_and_vars = optimizer.compute_gradients(cost, tvars)
clipped = [(tf.clip_by_value(grad, -5, 5), tvar) # gradient clipping
for grad, tvar in grads_and_vars]
train_op = optimizer.apply_gradients(clipped, global_step=global_step,
name="minimize_cost")
# ops to directly explore latent space
# defaults to prior z ~ N(0, I)
with tf.name_scope("latent_in"):
z_ = tf.placeholder_with_default(tf.random_normal([1, self.architecture[-1]]),
shape=[None, self.architecture[-1]],
name="latent_in")
x_reconstructed_ = composeAll(decoding)(z_)
return (x_in, dropout, z_mean, z_log_sigma, x_reconstructed,
z_, x_reconstructed_, cost, global_step, train_op)
def _buildGraph(self):
x_in = tf.placeholder(
tf.float32,
shape=[
None, # enables variable batch size
self.architecture[0]
],
name="x")
dropout_prob = tf.placeholder_with_default(1.,
shape=[],
name="dropout_prob")
# encoding / "recognition": q(z|x)
encoding = [
Dense("encoding", idx, hidden_size, dropout_prob,
self.nonlinearity)
for hidden_size, idx in zip(self.architecture[1:-1],
range(len(self.architecture) - 2))
]
# hidden layers reversed for function composition: outer -> inner
h_encoded = composeAll(reversed(encoding))(x_in)
self.layerSummaries(encoding, x_in)
# latent distribution parameterized by hidden encoding
# z ~ N(z_mean, z_sigma**2)
z_mean = Dense("z_mean",
len(self.architecture) - 2, self.architecture[-1],
1.)(h_encoded)
z_sigma = Dense("z_sigma",
len(self.architecture) - 2, self.architecture[-1],
1.)(h_encoded)
tf.summary.histogram('activations_z_mean', z_mean)
tf.summary.histogram('activations_z_sigma', z_sigma)
tf.summary.scalar('z_mean_mean-activations', tf.reduce_mean(z_mean))
tf.summary.scalar('z_sigma_mean-activations', tf.reduce_mean(z_sigma))
# kingma & welling: only 1 draw necessary as long as minibatch large enough (>100)
z = self.sampleGaussian(z_mean, z_sigma)
tf.summary.histogram('z_sampled', z)
# decoding (p(x|z)): assuming architecture symmetry
decoding = [
Dense("decoding", idx, hidden_size, dropout_prob,
self.nonlinearity) for hidden_size, idx in zip(
self.architecture[1:-1],
reversed(
range(
len(self.architecture) -
1, 2 * len(self.architecture) - 3, 1)))
]
# final reconstruction: prepend as outermost function
decoding.insert(
0,
Dense("x_decoding", 2 * len(self.architecture) - 3,
self.architecture[0], 1.0, self.output_activation))
x_reconstructed = tf.identity(composeAll(decoding)(z),
name="x_reconstructed")
self.layerSummaries(reversed(decoding), z)
mse_autoencoderTest = tf.losses.mean_squared_error(
x_reconstructed, x_in)
sqrt_mse_autoencoderTest = tf.sqrt(mse_autoencoderTest)
cosSim_autoencoderTest = self.cosSim(x_reconstructed, x_in)
tf.summary.scalar('autoencoderTest_MSE', mse_autoencoderTest)
tf.summary.scalar('autoencoderTest_sqrtMSE', sqrt_mse_autoencoderTest)
tf.summary.scalar('autoencoderTest_cosSim', cosSim_autoencoderTest)
# reconstruction loss: mismatch b/w x & x_reconstructed
# binary cross-entropy -- assumes x & p(x|z) are iid Bernoullis
rec_loss = VAE.crossEntropy(x_reconstructed, x_in)
#rec_loss = VAE.l2_loss(x_reconstructed, x_in)
tf.summary.scalar('cross_entropy', tf.reduce_mean(rec_loss))
# Kullback-Leibler divergence: mismatch b/w approximate vs. imposed/true posterior
kl_loss = VAE.kullbackLeibler(z_mean, z_sigma)
tf.summary.scalar('KL_divergence', tf.reduce_mean(kl_loss))
with tf.name_scope("l2_regularization"):
regularizers = [
tf.nn.l2_loss(var) for var in self.sesh.graph.get_collection(
"trainable_variables") if "weights" in var.name
]
l2_reg = self.lambda_l2_reg * tf.add_n(regularizers)
tf.summary.scalar('l2_reg', l2_reg)
with tf.name_scope("cost"):
# average over minibatch
cost = tf.reduce_mean(rec_loss + kl_loss, name="vae_cost")
tf.summary.scalar('cost_without_regularization', cost)
cost += l2_reg
tf.summary.scalar('cost', cost)
# optimization
total_updates = tf.Variable(0, trainable=False)
#with tf.name_scope("Adam_optimizer"):
with tf.name_scope("Adagrad_optimizer"):
#optimizer = tf.train.AdamOptimizer(self.learning_rate)
optimizer = tf.train.AdagradOptimizer(self.learning_rate)
tvars = tf.trainable_variables()
grads_and_vars = optimizer.compute_gradients(cost, tvars)
clipped = [
(tf.clip_by_value(grad, -5, 5), tvar) # gradient clipping
for grad, tvar in grads_and_vars
]
train_op = optimizer.apply_gradients(grads_and_vars,
global_step=total_updates,
name="minimize_cost")
# ops to directly explore latent space
# defaults to prior z ~ N(0, I)
with tf.name_scope("latent_in"):
z_ = tf.placeholder_with_default(
tf.random_normal([1, self.architecture[-1]]),
shape=[None, self.architecture[-1]],
name="latent_in")
x_reconstructed_ = composeAll(decoding)(z_)
return (x_in, dropout_prob, z_mean, z_sigma, x_reconstructed, z_,
x_reconstructed_, cost, total_updates, train_op,
mse_autoencoderTest, sqrt_mse_autoencoderTest,
cosSim_autoencoderTest)
def _buildGraph(self):
x_in = tf.placeholder(tf.float32,
shape=[None, self.architecture[0]],
name="x")
dropout = tf.placeholder_with_default(1., shape=[], name="dropout")
# encoding / "recognition": q(z|x)
encoding = [
Dense("encoding", hidden_size, dropout, self.nonlinearity)
for hidden_size in reversed(self.architecture[1:-1])
]
h_encoded = composeAll(encoding)(x_in)
# latent distribution parameterized by hidden encoding
# z ~ N(z_mean, np.exp(z_log_sigma)**2)
z_mean = Dense("z_mean", self.architecture[-1], dropout)(h_encoded)
z_log_sigma = Dense("z_log_sigma", self.architecture[-1],
dropout)(h_encoded)
z = self.sampleGaussian(z_mean, z_log_sigma)
# decoding / "generative": p(x|z)
decoding = [
Dense("decoding", hidden_size, dropout, self.nonlinearity)
for hidden_size in self.architecture[1:-1]
]
# final reconstruction: restore original dims, squash outputs [0, 1] ???
decoding.insert(
0,
Dense("x_decoding", self.architecture[0], dropout, self.squashing))
x_reconstructed = tf.identity(composeAll(decoding)(z),
name="x_reconstructed")
# reconstruction loss
rec_loss = VAE.crossEntropy(x_reconstructed, x_in)
# Kullback-Leibler divergense
kl_loss = VAE.kullbackLeibler(z_mean, z_log_sigma)
with tf.name_scope("l2_regularization"):
regularizers = [
tf.nn.l2_loss(var) for var in self.sesh.graph.get_collection(
"trainable_variables") if "weights" in var.name
]
l2_reg = self.lambda_l2_reg * tf.add_n(regularizers)
with tf.name_scope("cost"):
cost = tf.reduce_mean(rec_loss + kl_loss, name="vae_cost")
cost += l2_reg
# optimization
global_step = tf.Variable(0, trainable=False)
with tf.name_scope("Adam_optimizer"):
optimizer = tf.train.AdamOptimizer(self.learning_rate)
tvars = tf.trainable_variables()
grads_and_vars = optimizer.compute_gradients(cost, tvars)
clipped = [(tf.clip_by_value(grad, -5, 5), tvar)
for grad, tvar in grads_and_vars]
train_op = optimizer.apply_gradients(clipped,
global_step=global_step,
name="minimize_cost")
# ops to directly explore latent space, defaults to prior z ~ N(0, I)
# FIXME x_reconstructedとx_reconstructed_の違いって何？
with tf.name_scope("latent_in"):
z_ = tf.placeholder_with_default(
tf.random_normal([1, self.architecture[-1]]),
shape=[None, self.architecture[-1]],
name="latent_in")
x_reconstructed_ = composeAll(decoding)(z_)
return (x_in, dropout, z_mean, z_log_sigma, x_reconstructed, z_,
x_reconstructed_, cost, global_step, train_op)
def _buildGraph(self):
"""
x_in = tf.placeholder(tf.float32, shape=[None, # enables variable batch size
self.architecture[0]], name="x")
"""
# get data from input pipeline
filenames = []
for root, dirs, files in os.walk(self.data_path):
for file in files:
filenames.append(os.path.join(root, file))
f_tensor = tf.convert_to_tensor(filenames, dtype=tf.string)
filename_queue = tf.train.string_input_producer(f_tensor)
reader = tf.TextLineReader()
key, value = reader.read(filename_queue)
example = tf.decode_csv(value,
record_defaults=[[0]] * 1024,
field_delim='\t')
example = tf.stack(example)
"""
queue_batch=[]
for i in range(10):
key, value=reader.read(filename_queue)
example=tf.decode_csv(value,record_defaults=[[0]]*1024,field_delim='\t')
example=tf.stack(example)
queue_batch.append(example)
"""
x_q_in = tf.train.shuffle_batch([example],
batch_size=self.batch_size,
capacity=10000,
num_threads=20,
min_after_dequeue=200,
allow_smaller_final_batch=True,
enqueue_many=False)
x_q_in_norm = tf.to_float(x_q_in) / 255.
x_in = tf.placeholder_with_default(x_q_in_norm,
shape=(None, 1024),
name="x_in")
dropout = tf.placeholder_with_default(1., shape=[], name="dropout")
# encoding / "recognition": q(z|x)
encoding = [
Dense("encoding", hidden_size, dropout, self.nonlinearity)
# hidden layers reversed for function composition: outer -> inner
for hidden_size in reversed(self.architecture[1:-1])
]
h_encoded = composeAll(encoding)(x_in)
# latent distribution parameterized by hidden encoding
# z ~ N(z_mean, np.exp(z_log_sigma)**2)
z_mean = Dense("z_mean", self.architecture[-1], dropout)(h_encoded)
z_log_sigma = Dense("z_log_sigma", self.architecture[-1],
dropout)(h_encoded)
# kingma & welling: only 1 draw necessary as long as minibatch large enough (>100)
z = self.sampleGaussian(z_mean, z_log_sigma)
# decoding / "generative": p(x|z)
decoding = [
Dense("decoding", hidden_size, dropout, self.nonlinearity)
for hidden_size in self.architecture[1:-1]
] # assumes symmetry
# final reconstruction: restore original dims, squash outputs [0, 1]
decoding.insert(
0,
Dense( # prepend as outermost function
"x_decoding", self.architecture[0], dropout, self.squashing))
x_reconstructed = tf.identity(composeAll(decoding)(z),
name="x_reconstructed")
# reconstruction loss: mismatch b/w x & x_reconstructed
# binary cross-entropy -- assumes x & p(x|z) are iid Bernoullis
rec_loss = VAE.crossEntropy(x_reconstructed, x_in)
# Kullback-Leibler divergence: mismatch b/w approximate vs. imposed/true posterior
kl_loss = VAE.kullbackLeibler(z_mean, z_log_sigma)
with tf.name_scope("l2_regularization"):
regularizers = [
tf.nn.l2_loss(var) for var in self.sesh.graph.get_collection(
"trainable_variables") if "weights" in var.name
]
l2_reg = self.lambda_l2_reg * tf.add_n(regularizers)
with tf.name_scope("cost"):
# average over minibatch
cost = tf.reduce_mean(rec_loss + kl_loss, name="vae_cost")
cost += l2_reg
# optimization
global_step = tf.Variable(0, trainable=False)
with tf.name_scope("Adam_optimizer"):
optimizer = tf.train.AdamOptimizer(self.learning_rate)
tvars = tf.trainable_variables()
grads_and_vars = optimizer.compute_gradients(cost, tvars)
clipped = [
(tf.clip_by_value(grad, -5, 5), tvar) # gradient clipping
for grad, tvar in grads_and_vars
]
train_op = optimizer.apply_gradients(clipped,
global_step=global_step,
name="minimize_cost")
# ops to directly explore latent space
# defaults to prior z ~ N(0, I)
with tf.name_scope("latent_in"):
z_ = tf.placeholder_with_default(
tf.random_normal([1, self.architecture[-1]]),
shape=[None, self.architecture[-1]],
name="latent_in")
x_reconstructed_ = composeAll(decoding)(z_)
return (x_in, dropout, z_mean, z_log_sigma, x_reconstructed, z_,
x_reconstructed_, cost, global_step, train_op)
def _buildGraph(self):
x_in = tf.placeholder(tf.float32,
shape=(None, self.architecture[0]),
name="x_in")
dropout = tf.placeholder_with_default(1., shape=(), name="dropout")
# encoding / "recognition": q(z|x)
encoding = [
Dense("encoding",
hidden_size,
dropout,
self.nonlinearity,
initialization=self.initialization)
# hidden layers reversed for function composition:
# outer -> inner
for hidden_size in reversed(self.architecture[1:-1])
]
h_encoded = composeAll(encoding)(x_in)
# latent distribution parameterized by hidden encoding
# z ~ N(z_mean, np.exp(z_log_sigma)**2)
z_mean = Dense("z_mean",
self.architecture[-1],
dropout,
initialization=self.initialization)(h_encoded)
z_log_sigma = Dense("z_log_sigma",
self.architecture[-1],
dropout,
initialization=self.initialization)(h_encoded)
# kingma & welling: only 1 draw necessary as long as
# minibatch large enough (>100)
z = tf.identity(self.sampleGaussian(z_mean, z_log_sigma), name="z")
# this is also the entry point for latent space exploration
# decoding / "generative": p(x|z)
decoding = [
Dense("decoding",
hidden_size,
dropout,
self.nonlinearity,
initialization=self.initialization)
for hidden_size in self.architecture[1:-1]
]
# assumes symmetry
# final reconstruction: restore original dims
# prepend as outermost function
decoding.insert(
0,
Dense("x_decoding",
self.architecture[0],
dropout,
initialization=self.initialization))
outer_layer = composeAll(decoding)(z)
x_out = tf.nn.sigmoid(outer_layer, name="x_out")
# reconstruction loss: mismatch b/w x & x_out
# binary cross-entropy -- assumes x & p(x|z) are iid Bernoullis
rec_loss = VAE.crossEntropy(outer_layer, x_in)
# Kullback-Leibler divergence: mismatch b/w approximate
# vs. imposed/true posterior
kl_loss = VAE.kullbackLeibler(z_mean, z_log_sigma)
with tf.name_scope("l2_regularization"):
regularizers = [
tf.nn.l2_loss(var) for var in self.sesh.graph.get_collection(
"trainable_variables") if "weights" in var.name
]
l2_reg = self.lambda_l2_reg * tf.add_n(regularizers)
with tf.name_scope("cost"):
# average over minibatch plus l2 regularizer
cost = tf.reduce_mean(rec_loss + kl_loss, name="vae_cost") + l2_reg
# optimization
global_step = tf.Variable(0, trainable=False)
train_op = tf.contrib.layers.optimize_loss(
cost,
global_step,
self.learning_rate,
"Adam",
clip_gradients=self.grad_clipping,
name="trainer")
return (x_in, dropout, z_mean, z_log_sigma, x_out, z, cost,
global_step, train_op)