def _statistics_network(self, inputs, reuse=False):
""" Build the statistics network.
Args:
inputs: The concatinated realization drawn from the two random
vectors.
reuse (default: False): Whether the created variables can be
reused.
Returns:
[t_hidden, t_outputs]
t_hidden is a list of hidden layer output tensors.
t_outputs is the output tensor of the statistics network.
"""
# Hidden layers.
last_layer = inputs
t_hidden = []
for i in range(self.num_layers - 1):
with tf.variable_scope('hidden_%d' % (i), reuse=reuse):
hidden = layers.fully_connected(last_layer,
self._layer_sizes[i],
activation_fn=tf.nn.elu)
t_hidden.append(hidden)
last_layer = hidden
# Output layer.
with tf.variable_scope("output", reuse=reuse):
t_outputs = layers.fully_connected(last_layer,
1,
activation_fn=None)
return [t_hidden, t_outputs]
def _build_network(self, inputs):
"""This network builds the actual network, consisting of several 2D
convolutional layers and a final FC layer.
Args:
inputs: The tensor containing the inputs to the network (a 2D
image).
"""
tf.summary.image('input', inputs)
logger.info('Network input shape: [%s]' % (', '.join([str(e) for e in \
inputs.shape[1:]])))
# Convolutional layers.
last_layer = inputs
self._t_hidden = []
for i in range(self.num_layers - 1):
with tf.variable_scope('hidden_%d' % (i)):
if self.use_batchnorm:
use_bn = False
is_training = None
else:
use_bn = True
is_training = self._t_mode
hidden = layers.conv2d(last_layer,
self._conv_filters[i],
self._conv_kernels[i],
strides=self._conv_strides[i],
padding=self._conv_pads[i],
activation=tf.nn.relu,
pool_size=self._pool_kernels[i],
pool_strides=self._pool_strides[i],
pool_padding=self._pool_pads[i],
use_bn=use_bn,
is_training=is_training)
self._t_hidden.append(hidden)
last_layer = hidden
logger.info('Output shape after %d. convolutional layer: ' %
(i + 1) + '[%s]' %
(', '.join([str(e)
for e in last_layer.shape[1:]])))
curr_shape = int(np.prod(last_layer.shape[1:]))
last_layer = tf.reshape(last_layer, [-1, curr_shape])
# Final fully connected layer.
with tf.variable_scope("output"):
curr_shape = int(np.prod(last_layer.shape[1:]))
last_layer = tf.reshape(last_layer, [-1, curr_shape])
self._t_outputs = layers.fully_connected(last_layer,
shared.data.num_classes,
activation_fn=tf.nn.relu)
# For visualizations.
self._t_output_probs = tf.contrib.layers.softmax(self._t_outputs)
tf.summary.histogram('output', self._t_output_probs)
def _build_generator(self):
"""Builds the generator network, consisting of an initial fully-
connected layer followed by several strided transpose convolutional
layers.
"""
biases_initializer = tf.zeros_initializer() if self.use_biases \
else None
# The initial fully-connected layer, that transforms the latent input
# vectors in such a way, that it can be feeded into the deconv layers.
fc_out_shape = self._gen_fc_out_wof + [self._gen_filters[0]]
with tf.variable_scope('hidden_0'):
self._g_fc = layers.fully_connected(
self._g_inputs,
int(np.prod(fc_out_shape)),
activation_fn=tf.nn.relu,
weights_initializer=self._initializer,
biases_initializer=biases_initializer,
use_bn=True,
is_training=self._t_mode)
self._g_fc = tf.reshape(self._g_fc, [-1] + fc_out_shape)
last_layer = self._g_fc
# Transpose convolutional layers
self._g_hidden_deconv = []
for i in range(1, self.num_gen_layers - 1):
with tf.variable_scope('hidden_%d' % (i)):
hidden = layers.conv2d_transpose(
last_layer,
self._gen_filters[i],
self._gen_kernels[i - 1],
strides=self._gen_strides[i - 1],
padding=self._gen_pads[i - 1],
activation=tf.nn.relu,
use_bias=self.use_biases,
kernel_initializer=self._initializer,
use_bn=True,
is_training=self._t_mode)
self._g_hidden_deconv.append(hidden)
last_layer = hidden
# The generator output. Note, that it uses no batchnorm and a
# different activation function.
i = self.num_gen_layers - 1
with tf.variable_scope("output"):
self._g_outputs = layers.conv2d_transpose(
last_layer,
self._gen_filters[i],
self._gen_kernels[i - 1],
strides=self._gen_strides[i - 1],
padding=self._gen_pads[i - 1],
activation=tf.nn.tanh,
use_bias=self.use_biases,
kernel_initializer=self._initializer)
tf.summary.image('fake', tf.reshape(self._g_outputs, \
[-1] + shared.data.in_shape))
def _build_discriminator(self, inputs, reuse=False):
"""Build the discriminator network, which consists of consecutive
strided convolutional layers (without pooling), followed by a fully-
connected network with a single output.
Args:
inputs: The input tensor to the discriminator.
reuse: Whether the created variables can be reused.
Returns:
[d_hidden, d_outputs]
d_hidden is a list of hidden layer output tensors.
d_outputs is the output tensor of the discriminator.
"""
biases_initializer = tf.zeros_initializer() if self.use_biases \
else None
# Convolutional layers.
last_layer = inputs
d_hidden = []
for i in range(self.num_dis_layers - 1):
with tf.variable_scope('hidden_%d' % (i), reuse=reuse):
# Note, that we don't use batchnorm in the first layer of the
# discriminator.
if i == 0:
use_bn = False
is_training = None
else:
use_bn = True
is_training = self._t_mode
hidden = layers.conv2d(last_layer,
self._dis_filters[i],
self._dis_kernels[i],
strides=self._dis_strides[i],
padding=self._dis_pads[i],
activation=tf.nn.leaky_relu,
kernel_initializer=self._initializer,
use_bias=self.use_biases,
use_bn=use_bn,
is_training=is_training)
d_hidden.append(hidden)
last_layer = hidden
# Final fully connected layer.
with tf.variable_scope("output", reuse=reuse):
curr_sample_size = int(np.prod(last_layer.shape[1:]))
last_layer = tf.reshape(last_layer, [-1, curr_sample_size])
d_outputs = layers.fully_connected(
last_layer,
1,
activation_fn=None,
weights_initializer=self._initializer,
biases_initializer=biases_initializer)
return [d_hidden, d_outputs]
def build(self):
"""Build the network, such that we can use it to run training or
inference."""
self._is_build = True
msg = 'Building a ' + str(self.num_layers) + '-layer fully-' + \
'connected autoencoder for the dataset: ' + \
shared.data.get_identifier()
logger.info(msg)
# Note, that we need to remember the graph, in case we want to
# construct multiple instances of this class (or multiple tensorflow
# graphs in general). Because in this case, we can't just refer to the
# default graph.
tf.reset_default_graph()
self._graph = tf.Graph()
with self._graph.as_default() as g:
# Note, that we have to use name_scope, not variable_scope, as we
# do want to keep the same variable names across instances of this
# class (otherwise, we couldn't restore the graph as simply from a
# checkpoint). However, we want to define a named scope for
# Tensorboard visualizations, that's why we use name_scope. Note,
# that this will only affect the readibility within a Tensorboard
# session. We cannot write multiple graphs within the same
# Tensorboard session.
with g.name_scope(self._scope_name) as scope:
self._build_datasets()
# Network Inputs.
self._t_inputs = tf.placeholder_with_default(
self._t_ds_inputs, shape=[None, self._layer_sizes[0]],
name='inputs')
self._t_learning_rate = tf.placeholder(tf.float32, shape=[])
batch_size = tf.shape(self._t_inputs)[0]
last_layer = self._t_inputs
# Hidden layers.
self._t_hidden = []
for i in range(1, self.num_layers-1):
with tf.variable_scope('hidden_%d' % (i)):
hidden = layers.fully_connected(last_layer, \
self._layer_sizes[i], activation_fn=tf.nn.relu)
self._t_hidden.append(hidden)
last_layer = hidden
# Input reconstructions.
with tf.variable_scope("output"):
self._t_outputs = layers.fully_connected(last_layer, \
int(self._layer_sizes[-1]), activation_fn=tf.nn.relu)
img_dataset, _ = shared.data.is_image_dataset()
if img_dataset:
tf.summary.image('reconstruction', \
tf.reshape(self._t_outputs, \
[-1] + shared.data.in_shape))
# Compute loss: Reconstruction Error
diff = self._t_inputs - self._t_outputs
reconstruction_err = tf.nn.l2_loss(diff) / \
float(self._layer_sizes[0])
reconstruction_err /= tf.to_float(batch_size)
tf.losses.add_loss(reconstruction_err)
self._t_loss = tf.losses.get_total_loss()
tf.summary.scalar('loss', self._t_loss)
self._t_summaries = tf.summary.merge_all()
self._t_global_step = tf.get_variable('global_step', shape=(),
initializer=tf.constant_initializer(0), trainable=False)
self._scope = scope