def __init__(self,
rlr, alr, weight_scale,
encoder_outfuns, encoder_layers,
decoder_outfuns, decoder_layers,
adversarial_outfuns, adversarial_layers):
self.rlr = rlr
self.alr = alr
self.encoder_layers = encoder_layers
self.decoder_layers = decoder_layers
self.encoder = MLP(scope="Encoder",
weight_scale=weight_scale,
outfuns=encoder_outfuns,
layers_lens=encoder_layers)
self.decoder = MLP(scope="Decoder",
weight_scale=weight_scale,
outfuns=decoder_outfuns,
layers_lens=decoder_layers)
self.adversarial = MLP(scope="Adversarial",
weight_scale=weight_scale,
outfuns=adversarial_outfuns,
layers_lens=adversarial_layers)
self.make_graph()
def __init__(self, rlr, weight_scale, autoencoder_outfuns,
autoencoder_layers, autoencoder_dropouts=None,
autoencoder_bn=None, bn_decay=None,
autoencoder_convs=None, autoencoder_deconvs=None,
autoencoder_strides=None, autoencoder_copy_from=None):
self.rlr = rlr
self.autoencoder_layers = autoencoder_layers
self.autoencoder_layers = autoencoder_layers
if bn_decay is None: bn_decay = 0.99
self.autoencoder = MLP(scope="autoencoder",
weight_scale=weight_scale,
bn_decay=bn_decay,
outfuns=autoencoder_outfuns,
layers_lens=autoencoder_layers,
drop_out=autoencoder_dropouts,
convs=autoencoder_convs,
deconvs=autoencoder_deconvs,
strides=autoencoder_strides,
batch_norms=autoencoder_bn)
self.make_graph()
autoencoder_outfuns += [tf.nn.relu, tf.nn.relu, tf.nn.relu]
autoencoder_copy_from += [None, None, None]
autoencoder_dropout += [True, True, True]
autoencoder_batch_norms += [True, True, True]
autoencoder_layers += [512, 256, n_mnist_pixels]
autoencoder_outfuns += [tf.nn.relu, tf.nn.relu, tf.tanh]
autoencoder_copy_from += [2, 1, 0]
autoencoder_dropout += [True, True, False]
autoencoder_batch_norms += [True, True, True]
autoencoder = MLP(scope="autoencoder",
lr=lr,
bn_decay=decay,
weight_scale=weight_scale,
outfuns=autoencoder_outfuns,
layers_lens=autoencoder_layers,
drop_out=autoencoder_dropout,
copy_from=autoencoder_copy_from,
batch_norms=autoencoder_batch_norms)
data_sample = tf.placeholder(tf.float32,
[num_samples, autoencoder_layers[0]])
decoded_patterns = autoencoder.update(data_sample,
drop_out=drop_out,
phase_train=phase_train)
#---------------------------------------------------------------------------
R_loss = tf.sqrt(tf.reduce_mean(tf.pow(data_sample - decoded_patterns, 2)))
#---------------------------------------------------------------------------
train = autoencoder.train(R_loss)
#---------------------------------------------------------------------------
autoencoder_layers += [[7, 7, 10], [7, 7, 5], [14, 14, 3], [28, 28, 1]]
autoencoder_outfuns += [tf.nn.relu, tf.nn.relu, tf.nn.relu, tf.tanh]
autoencoder_convs += [None, None, None, None]
autoencoder_deconvs += [
None, [20, 20, 5, 10], [12, 12, 3, 5], [3, 3, 1, 3]
]
autoencoder_copy_from += [None, 2, 1, 0]
autoencoder_strides += [None, 1, 2, 2]
autoencoder_bn += [True, True, True, True]
autoencoder = MLP(scope="Autoencoder",
lr=lr,
bn_decay=decay,
weight_scale=weight_scale,
convs=autoencoder_convs,
deconvs=autoencoder_deconvs,
strides=autoencoder_strides,
outfuns=autoencoder_outfuns,
copy_from=autoencoder_copy_from,
layers_lens=autoencoder_layers,
batch_norms=autoencoder_bn)
data_sample = tf.placeholder(tf.float32,
[num_samples] + autoencoder_layers[0])
reconstructed_sample = autoencoder.update(data_sample,
phase_train=phase_train)
#---------------------------------------------------------------------------
R_loss = tf.reduce_mean(tf.pow(data_sample - reconstructed_sample, 2.0))
#---------------------------------------------------------------------------
R_train = autoencoder.train(R_loss,
optimizer=tf.train.GradientDescentOptimizer)
class AAE:
eps = 1.0e-5
def __init__(self,
rlr, alr, weight_scale,
encoder_outfuns, encoder_layers,
decoder_outfuns, decoder_layers,
adversarial_outfuns, adversarial_layers):
self.rlr = rlr
self.alr = alr
self.encoder_layers = encoder_layers
self.decoder_layers = decoder_layers
self.encoder = MLP(scope="Encoder",
weight_scale=weight_scale,
outfuns=encoder_outfuns,
layers_lens=encoder_layers)
self.decoder = MLP(scope="Decoder",
weight_scale=weight_scale,
outfuns=decoder_outfuns,
layers_lens=decoder_layers)
self.adversarial = MLP(scope="Adversarial",
weight_scale=weight_scale,
outfuns=adversarial_outfuns,
layers_lens=adversarial_layers)
self.make_graph()
def make_graph(self):
self.data_sample = tf.placeholder(tf.float32,
[None, self.encoder_layers[0]])
self.prior_sample = tf.placeholder(tf.float32,
[None, self.decoder_layers[0]])
self.drop_out = tf.placeholder(tf.float32, ())
self.phase_train = tf.placeholder(tf.bool, ())
self.hidden_patterns = self.encoder.update(self.data_sample,
drop_out=self.drop_out, phase_train=self.phase_train)
self.decoded_patterns = self.decoder.update(self.hidden_patterns,
drop_out=self.drop_out, phase_train=self.phase_train)
self.prior_generated_patterns = self.decoder.update(
self.prior_sample, drop_out=self.drop_out,
phase_train=self.phase_train)
self.D_probs = self.adversarial.update(self.prior_sample,
drop_out=self.drop_out, phase_train=self.phase_train)
self.G_probs = self.adversarial.update(self.hidden_patterns,
drop_out=self.drop_out, phase_train=self.phase_train)
self.losses()
self.optimize()
def losses(self):
self.R_loss = tf.losses.mean_squared_error(self.data_sample,
self.decoded_patterns)
self.D_loss = tf.reduce_mean(-tf.log(self.D_probs + self.eps)) \
- tf.reduce_mean(tf.log(1.0 - self.G_probs + self.eps))
self.G_loss = tf.reduce_mean(-tf.log(self.G_probs + self.eps))
def optimize(self):
self.ER_train = self.encoder.train(self.R_loss, lr=self.rlr)
self.DR_train = self.decoder.train(self.R_loss, lr=self.rlr)
self.D_train = self.adversarial.train(self.D_loss, lr=self.alr)
self.G_train = self.encoder.train(self.G_loss, lr=self.alr)
def train_step(self, session, curr_data_sample,
curr_prior_sample, dropout=0.3):
# reconstruction step -- (minimize reconstruction error)
# data_sample -> encoder -> hidden_patterns
# -> decoder -> decoded_patterns
current_decoded_patterns, r_loss, _, _ = session.run(
[self.decoded_patterns, self.R_loss, self.ER_train,
self.DR_train], feed_dict={
self.data_sample:curr_data_sample,
self.drop_out: 1.0, self.phase_train: True})
# adversarial step -- (minimize discrimination error)
# data_sample -> encoder -> hidden_patterns
# -> adversarial -> G_props
# prior_sample -> adversarial -> D_props
d_loss, _ = session.run([self.D_loss, self.D_train],
feed_dict={self.data_sample:curr_data_sample,
self.prior_sample:curr_prior_sample,
self.drop_out: dropout, self.phase_train: True})
# adversarial step -- (maximize discrimination error)
# data_sample -> encoder -> hidden_patterns
# -> adversarial -> G_props
g_loss, _ = session.run([self.G_loss, self.G_train],
feed_dict={self.data_sample:curr_data_sample,
self.drop_out: dropout, self.phase_train: True})
return current_decoded_patterns, r_loss, d_loss, g_loss
def test_step(self, session, grid, test_data):
# Current generation of image patterns in response to
# the presentation of a 2D grid of values to the two
# hidden units
grid_patterns = session.run(self.prior_generated_patterns,
feed_dict={self.prior_sample:grid, self.drop_out: 1.0,
self.phase_train: False})
# Current activation of hidden units in response to the
# presentation of data images
curr_hidden_patterns = session.run(self.hidden_patterns,
feed_dict={self.data_sample:test_data, self.drop_out: 1.0,
self.phase_train: False})
return grid_patterns, curr_hidden_patterns
with graph.as_default():
#-------------------------------------------------------------------------------
drop_out = tf.placeholder(tf.float32, ())
phase_train = tf.placeholder(tf.bool)
#---------------------------------------------------------------------------
# Encoder
#### network init
encoder_layers = [n_mnist_pixels, 1000, 1000, 2]
encoder_dropout =[False, False, False]
encoder_bn = [False, False, False]
encoder_outfuns =[tf.nn.relu, tf.nn.relu, linear]
encoder = MLP(scope="Encoder",
lr=rlr,
bn_decay=decay,
weight_scale=weight_scale,
batch_norms=encoder_bn,
outfuns=encoder_outfuns,
drop_out=encoder_dropout,
layers_lens=encoder_layers)
data_sample = tf.placeholder(tf.float32, [num_samples, encoder_layers[0]])
#### training branch
hidden_patterns = encoder.update(data_sample, drop_out=drop_out, phase_train=phase_train)
#### test branch
data_test = tf.placeholder(tf.float32, [tests, encoder_layers[0]])
test_hidden_patterns = encoder.update(data_test, drop_out=drop_out, phase_train=phase_train)
print "Encoder tree has been built"
#---------------------------------------------------------------------------
# Decoder
#### network init
decoder_layers = [2, 1000, 1000, n_mnist_pixels]
class CAE:
def __init__(self, rlr, weight_scale, autoencoder_outfuns,
autoencoder_layers, autoencoder_dropouts=None,
autoencoder_bn=None, bn_decay=None,
autoencoder_convs=None, autoencoder_deconvs=None,
autoencoder_strides=None, autoencoder_copy_from=None):
self.rlr = rlr
self.autoencoder_layers = autoencoder_layers
self.autoencoder_layers = autoencoder_layers
if bn_decay is None: bn_decay = 0.99
self.autoencoder = MLP(scope="autoencoder",
weight_scale=weight_scale,
bn_decay=bn_decay,
outfuns=autoencoder_outfuns,
layers_lens=autoencoder_layers,
drop_out=autoencoder_dropouts,
convs=autoencoder_convs,
deconvs=autoencoder_deconvs,
strides=autoencoder_strides,
batch_norms=autoencoder_bn)
self.make_graph()
def make_graph(self):
self.data_sample = tf.placeholder(tf.float32,
[None] + self.autoencoder_layers[0])
self.drop_out = tf.placeholder(tf.float32, ())
self.phase_train = tf.placeholder(tf.bool, ())
self.decoded_patterns = self.autoencoder.update(self.data_sample,
drop_out=self.drop_out, phase_train=self.phase_train)
self.losses()
self.optimize()
def losses(self):
self.R_loss = tf.losses.mean_squared_error(self.data_sample,
self.decoded_patterns)
def optimize(self):
self.AER_train = self.autoencoder.train(self.R_loss, lr=self.rlr)
def train_step(self, session, curr_data_sample, dropout=0.3):
# reconstruction step -- (minimize reconstruction error)
# data_sample -> autoencoder -> hidden_patterns
# -> autoencoder -> decoded_patterns
current_decoded_patterns, r_loss, _ = session.run(
[self.decoded_patterns, self.R_loss, self.AER_train],
feed_dict={ self.data_sample:curr_data_sample,
self.drop_out: 1.0, self.phase_train: True})
return current_decoded_patterns, r_loss
def test_step(self, session, test_data):
# Current decoding in response to the
# presentation of data images
curr_decoded_patterns = session.run( self.decoded_patterns,
feed_dict={self.data_sample:test_data, self.drop_out:
1.0, self.phase_train: False})
return curr_decoded_patterns
def __init__(self,
rlr,
alr,
weight_scale,
encoder_outfuns,
encoder_layers,
decoder_outfuns,
decoder_layers,
adversarial_outfuns,
adversarial_layers,
encoder_dropouts=None,
decoder_dropouts=None,
adversarial_dropouts=None,
encoder_bn=None,
decoder_bn=None,
adversarial_bn=None,
bn_decay=None,
encoder_convs=None,
decoder_convs=None,
adversarial_convs=None,
encoder_deconvs=None,
decoder_deconvs=None,
adversarial_deconvs=None,
encoder_strides=None,
decoder_strides=None,
adversarial_strides=None,
encoder_copy_from=None,
decoder_copy_from=None,
adversarial_copy_from=None):
self.rlr = rlr
self.alr = alr
self.encoder_layers = encoder_layers
self.decoder_layers = decoder_layers
if bn_decay is None: bn_decay = 0.99
self.encoder = MLP(scope="Encoder",
weight_scale=weight_scale,
bn_decay=bn_decay,
outfuns=encoder_outfuns,
layers_lens=encoder_layers,
drop_out=encoder_dropouts,
convs=encoder_convs,
deconvs=encoder_deconvs,
strides=encoder_strides,
batch_norms=encoder_bn)
self.decoder = MLP(scope="Decoder",
weight_scale=weight_scale,
bn_decay=bn_decay,
outfuns=decoder_outfuns,
layers_lens=decoder_layers,
drop_out=decoder_dropouts,
convs=decoder_convs,
deconvs=decoder_deconvs,
strides=decoder_strides,
batch_norms=decoder_bn)
self.adversarial = MLP(scope="Adversarial",
weight_scale=weight_scale,
bn_decay=bn_decay,
outfuns=adversarial_outfuns,
layers_lens=adversarial_layers,
drop_out=adversarial_dropouts,
convs=adversarial_convs,
deconvs=adversarial_deconvs,
strides=adversarial_strides,
batch_norms=adversarial_bn)
self.make_graph()