def add_tower_loss(self, labels, batch_size=None, scope=None):
"""Adds all losses for the model.
Note the final loss is not returned. Instead, the list of losses are collected
by slim.losses. The losses are accumulated in tower_loss() and summed to
calculate the total loss.
Args:
logits: List of logits from inference(). Each entry is a 2-D float Tensor.
labels: Labels from distorted_inputs or inputs(). 1-D tensor of shape [batch_size]
batch_size: integer
scope: tower scope of losses to add, ie 'tower_0/', defaults to last added tower if None
"""
if not batch_size:
batch_size = FLAGS.batch_size
tower = self.tower(scope)
# Reshape the labels into a dense Tensor of
# shape [FLAGS.batch_size, num_classes].
sparse_labels = tf.reshape(labels, [batch_size, 1])
indices = tf.reshape(tf.range(batch_size), [batch_size, 1])
concated = tf.concat(1, [indices, sparse_labels])
num_classes = tower.logits.get_shape()[-1].value
dense_labels = tf.sparse_to_dense(concated, [batch_size, num_classes], 1.0, 0.0)
# Cross entropy loss for the main softmax prediction.
losses.softmax_cross_entropy(
tower.logits, dense_labels, label_smoothing=0.1, weight=1.0)
def add_tower_loss(self, labels, batch_size=None, scope=None):
"""Adds all losses for the model.
Args:
logits: List of logits from inference(). Each entry is a 2-D float Tensor.
labels: Labels from distorted_inputs or inputs(). 1-D tensor of shape [batch_size]
batch_size: integer
"""
if not batch_size:
batch_size = FLAGS.batch_size
tower = self.tower(scope)
# Reshape the labels into a dense Tensor of
# shape [FLAGS.batch_size, num_classes].
sparse_labels = tf.reshape(labels, [batch_size, 1])
indices = tf.reshape(tf.range(batch_size), [batch_size, 1])
concated = tf.concat(1, [indices, sparse_labels])
num_classes = tower.logits.get_shape()[-1].value
dense_labels = tf.sparse_to_dense(concated, [batch_size, num_classes],
1.0, 0.0)
# Cross entropy loss for the main softmax prediction.
losses.softmax_cross_entropy(tower.logits,
dense_labels,
label_smoothing=0.1,
weight=1.0)
def model_function(features, targets, mode):
# 1st hidden layer
hlayer = layers.fully_connected(inputs=features,
num_outputs=50,
activation_fn=tf.sigmoid) # Sigmoid perceptrons
# Shallow neural network because there is only 1 hidden layer
outputs = layers.fully_connected(inputs=hlayer,
num_outputs=10, # 10 perceptrons in output layer for 10 numbers (0 to 9)
activation_fn=None) # Use "None" as activation function specified in "softmax_cross_entropy" loss
# Calculate loss using cross-entropy error; also use the 'softmax' activation function
loss = losses.softmax_cross_entropy (outputs, targets)
optimizer = layers.optimize_loss(
loss=loss,
global_step=tf.contrib.framework.get_global_step(),
learning_rate=0.001,
optimizer="SGD")
# Class of output (i.e., predicted number) corresponds to the perceptron returning the highest fractional value
# Returning both fractional values and corresponding labels
probs = tf.nn.softmax(outputs)
return {'probs':probs, 'labels':tf.argmax(probs, 1)}, loss, optimizer
def model_function(features, targets, mode):
hlayers = layers.stack(
features,
layers.fully_connected, [1000, 100, 50, 20],
activation_fn=tf.nn.relu,
weights_regularizer=layers.l1_l2_regularizer(1.0, 2.0),
weights_initializer=layers.xavier_initializer(uniform=True, seed=100))
# hidden layers have to be fully connected for best performance. So, no option in tensorflow for
# non-fully connected layers; need to write custom code to do that
outputs = layers.fully_connected(
inputs=hlayers,
num_outputs=10, # 10 perceptrons in output layer for 10 numbers (0 to 9)
activation_fn=None
) # Use "None" as activation function specified in "softmax_cross_entropy" loss
# Calculate loss using cross-entropy error; also use the 'softmax' activation function
loss = losses.softmax_cross_entropy(outputs, targets)
optimizer = layers.optimize_loss(
loss=loss,
global_step=tf.contrib.framework.get_global_step(),
learning_rate=0.8,
optimizer="SGD")
# Class of output (i.e., predicted number) corresponds to the perceptron returning the highest fractional value
# Returning both fractional values and corresponding labels
probs = tf.nn.softmax(outputs)
return {'probs': probs, 'labels': tf.argmax(probs, 1)}, loss, optimizer
def model_function(features, targets, mode):
# Configure the single layer perceptron model
hlayer1 = layers.fully_connected(inputs=features,
num_outputs=20,
activation_fn=tf.sigmoid)
hlayer2 = layers.fully_connected(inputs=hlayer1,
num_outputs=10,
activation_fn=tf.sigmoid)
outputs = layers.fully_connected(inputs=hlayer2,
num_outputs=10,
activation_fn=None)
# Calculate loss using mean squared error
loss = losses.softmax_cross_entropy(outputs, targets)
# Create an optimizer for minimizing the loss function
optimizer = layers.optimize_loss(
loss=loss,
global_step=tf.contrib.framework.get_global_step(),
learning_rate=0.5,
optimizer="SGD")
probs = tf.nn.softmax(outputs)
return {'probs':probs, 'labels':tf.arg_max(probs,1)}, loss, optimizer
def model_function(features, targets, mode):
# Two hidden layers - 20,10 = # perceptrons in layer1, layer2. Both have ReLU activation
# More concise syntax
hlayers = layers.stack(features,
layers.fully_connected, [20, 10],
activation_fn=tf.nn.relu)
# hidden layers have to be fully connected for best performance. So, no option in tensorflow for
# non-fully connected layers; need to write custom code to do that
outputs = layers.fully_connected(
inputs=hlayers,
num_outputs=10, # 10 perceptrons in output layer for 10 numbers (0 to 9)
activation_fn=None
) # Use "None" as activation function specified in "softmax_cross_entropy" loss
# Calculate loss using cross-entropy error; also use the 'softmax' activation function
loss = losses.softmax_cross_entropy(outputs, targets)
optimizer = layers.optimize_loss(
loss=loss,
global_step=tf.contrib.framework.get_global_step(),
learning_rate=0.001,
optimizer="SGD")
# Class of output (i.e., predicted number) corresponds to the perceptron returning the highest fractional value
# Returning both fractional values and corresponding labels
probs = tf.nn.softmax(outputs)
return {'probs': probs, 'labels': tf.argmax(probs, 1)}, loss, optimizer
def model_function(features, targets, mode):
# don't need one-hot encoding since target is already in one-hot format
# sigmoid also will work although the interpretability is difficult;
# The output with the max. value corresponds to the 'class' - whether sigmoid or softmax
outputs = layers.fully_connected(
inputs=features,
num_outputs=10, # 10 perceptrons for 10 numbers (0 to 9)
activation_fn=None
) # Use "None" as activation function specified in "softmax_cross_entropy" loss
# layer gives direct/plain outputs - linear activation. To compute losses, we use softmax on top of plain outputs
# Calculate loss using cross-entropy error; also use the 'softmax' activation function
# softmax and cross-entropy combined together to handle log(0) and other border-case issues
loss = losses.softmax_cross_entropy(outputs, targets)
optimizer = layers.optimize_loss(
loss=loss,
# step is not an integer but a wrapper around it, just as Java has 'Integer' on top of 'int'
global_step=tf.contrib.framework.get_global_step(),
learning_rate=0.5,
optimizer="SGD")
# Class of output (i.e., predicted number) corresponds to the perceptron returning the highest fractional value
# Returning both fractional values and corresponding labels
probs = tf.nn.softmax(outputs)
return {'probs': probs, 'labels': tf.argmax(probs, 1)}, loss, optimizer
def model_function(features, targets, mode):
# Configure the single layer perceptron model
hlayers = layers.stack(features,
layers.fully_connected, [20, 10],
activation_fn=tf.nn.relu,
weights_initializer=layers.xavier_initializer(
True, seed=100))
outputs = layers.fully_connected(inputs=hlayers,
num_outputs=10,
activation_fn=None)
# Calculate loss using mean squared error
loss = losses.softmax_cross_entropy(outputs, targets)
# Create an optimizer for minimizing the loss function
optimizer = layers.optimize_loss(
loss=loss,
global_step=tf.contrib.framework.get_global_step(),
learning_rate=0.05,
optimizer="SGD")
probs = tf.nn.softmax(outputs)
return {'probs': probs, 'labels': tf.arg_max(probs, 1)}, loss, optimizer
def _softmax_entropy(probabilities, targets):
return metric_ops.streaming_mean(
losses.softmax_cross_entropy(probabilities, targets))
def model_function(features, targets, mode):
#input layer
#Reshape features to 4-D tensor: [batch_size, width, height, channels]
# MNIST images are 28x28 pixels, and have one color channel
#batch_size corresponds to number of images: -1 represents compute the number of images automatically
input_layer = tf.reshape(features, [-1, 28, 28, 1])
# Convolutional Layer #1
# Computes 32 features using a 5x5 filter with ReLU activation.
# Padding is added to preserve width and height.
# Input Tensor Shape: [batch_size, 28, 28, 1]
# Output Tensor Shape: [batch_size, 28, 28, 32]
conv1 = layers.conv2d(inputs=input_layer,
num_outputs=32,
kernel_size=[5, 5],
stride=1,
padding="SAME",
activation_fn=tf.nn.relu)
# Pooling Layer #1
# First max pooling layer with a 2x2 filter and stride of 2
# Input Tensor Shape: [batch_size, 28, 28, 32]
# Output Tensor Shape: [batch_size, 14, 14, 32]
pool1 = layers.max_pool2d(inputs=conv1, kernel_size=[2, 2], stride=2)
# Convolutional Layer #2
# Computes 64 features using a 5x5 filter.
# Padding is added to preserve width and height.
# Input Tensor Shape: [batch_size, 14, 14, 32]
# Output Tensor Shape: [batch_size, 14, 14, 64]
conv2 = layers.conv2d(inputs=pool1,
num_outputs=64,
kernel_size=[5, 5],
stride=1,
padding="SAME",
activation_fn=tf.nn.relu)
# Pooling Layer #2
# Second max pooling layer with a 2x2 filter and stride of 2
# Input Tensor Shape: [batch_size, 14, 14, 64]
# Output Tensor Shape: [batch_size, 7, 7, 64]
pool2 = layers.max_pool2d(inputs=conv2, kernel_size=[2, 2], stride=2)
# Flatten tensor into a batch of vectors
# Input Tensor Shape: [batch_size, 7, 7, 64]
# Output Tensor Shape: [batch_size, 7 * 7 * 64]
pool2_flat = tf.reshape(pool2, [-1, 7 * 7 * 64])
# Fully connected Layers with 100, 20 neurons
# Input Tensor Shapuntitled0.e: [batch_size, 14 * 14 * 32]
# Output Tensor Shape: [batch_size, 10]
fclayers = layers.stack(pool2_flat,
layers.fully_connected, [100, 20],
activation_fn=tf.nn.relu)
outputs = layers.fully_connected(inputs=fclayers,
num_outputs=10,
activation_fn=None)
# Calculate loss using mean squared error
loss = losses.softmax_cross_entropy(outputs, targets)
# Create an optimizer for minimizing the loss function
optimizer = layers.optimize_loss(
loss=loss,
global_step=tf.contrib.framework.get_global_step(),
learning_rate=0.8,
optimizer="SGD")
probs = tf.nn.softmax(outputs)
return {'probs': probs, 'labels': tf.arg_max(probs, 1)}, loss, optimizer
def _softmax_entropy(probabilities, targets):
return metric_ops.streaming_mean(losses.softmax_cross_entropy(
probabilities, targets))
def model_function(features, targets, mode):
# input layer
# Reshape features to 4-D tensor (55000x28x28x1)
# MNIST images are 28x28 pixels
# batch size corresponds to number of images: -1 represents ' compute the # images automatically (55000)'
# +1 represents the # channels. Here #channels =1 since grey image. For color image, #channels=3
input_layer = tf.reshape(features, [-1, 28, 28, 1])
# Computes 32 features using a 5x5 filter
# Padding is added to preserve width
# Input Tensor Shape: [batch_size,28,28,1]
# Output Tensor Shape: [batch_size,28,28,32]
conv1 = layers.conv2d(
inputs=input_layer,
num_outputs=32,
kernel_size=[5, 5],
stride=1,
padding=
"SAME", # do so much padding such that the feature map is same size as input
activation_fn=tf.nn.relu)
# Pooling layer 1
# Pooling layer ith a 2x2 filter and stride 2
# Input shape: [batch_size,28,28,32]
# Output shape: [batch_size,14,14,32]
pool1 = layers.max_pool2d(inputs=conv1, kernel_size=[2, 2], stride=2)
# Convolution layer 2
# Input: 14 x 14 x 32 (32 channels here)
# Output: 14 x 14 x 64 (32 features/patches fed to each perceptron; discovering 64 features)
conv2 = layers.conv2d(
inputs=pool1,
num_outputs=64,
kernel_size=[5, 5],
stride=1,
padding=
"SAME", # do so much padding such that the feature map is same size as input
activation_fn=tf.nn.relu)
# Pooling layer 2
# Input: 14 x14 x 64
# Output: 7 x 7 x 64
pool2 = layers.max_pool2d(inputs=conv2, kernel_size=[2, 2], stride=2)
# Flatten the pool2 to feed to the 1st layer of fully connected layers
# Input size: [batch_size,7,7,64]
# Output size: [batch_size, 7x7x64]
pool2_flat = tf.reshape(pool2, [-1, 7 * 7 * 64])
# Connected layers with 100, 20 neurons
# Input shape: [batch_size, 7x7x64]
# Output shape: [batch_size, 10]
fclayers = layers.stack(
pool2_flat,
layers.fully_connected, [100, 20],
activation_fn=tf.nn.relu,
weights_regularizer=layers.l1_l2_regularizer(1.0, 2.0),
weights_initializer=layers.xavier_initializer(uniform=True, seed=100))
outputs = layers.fully_connected(
inputs=fclayers,
num_outputs=10, # 10 perceptrons in output layer for 10 numbers (0 to 9)
activation_fn=None
) # Use "None" as activation function specified in "softmax_cross_entropy" loss
# Calculate loss using cross-entropy error; also use the 'softmax' activation function
loss = losses.softmax_cross_entropy(outputs, targets)
optimizer = layers.optimize_loss(
loss=loss,
global_step=tf.contrib.framework.get_global_step(),
learning_rate=0.1,
optimizer="SGD")
# Class of output (i.e., predicted number) corresponds to the perceptron returning the highest fractional value
# Returning both fractional values and corresponding labels
probs = tf.nn.softmax(outputs)
return {'probs': probs, 'labels': tf.argmax(probs, 1)}, loss, optimizer
def ladder_network(X,
y,
hidden_units,
activation='relu',
gaussian_stddev=1.0,
random_state=42,
scope=None):
"""Fully supervised ladder network."""
with tf.variable_scope(scope, 'ladder_network', [X, y]) as scope:
X = tf.convert_to_tensor(X)
y = tf.convert_to_tensor(y)
n_layers = len(hidden_units)
n_classes = tensor_utils.get_shape(y)[-1]
noisy_input = noise_ops.noise_layer(X,
noise_type='gaussian',
gaussian_stddev=gaussian_stddev,
random_state=random_state)
# First run the corrupt data through the corrupted encoder
encoder = noisy_input
for layer_number, n_hidden_units in enumerate(hidden_units):
encoder = fully_connected_encoder(
encoder,
n_hidden_units,
activation=activation,
noise_type='gaussian',
gaussian_stddev=gaussian_stddev,
random_state=random_state,
hidden_outputs_collections=NOISY_ENCODER_HIDDEN_STATE,
outputs_collections=NOISY_ENCODER_OUTPUTS,
scope='encoding_layer_{:d}'.format(layer_number))
# noisy classification head
noisy_logits = fully_connected_encoder(
encoder,
n_hidden_units=n_classes,
activation='identity',
noise_type='gaussian',
gaussian_stddev=gaussian_stddev,
random_state=random_state,
hidden_outputs_collections=NOISY_ENCODER_HIDDEN_STATE,
outputs_collections=NOISY_ENCODER_OUTPUTS,
scope='encoding_layer_{:d}'.format(n_layers))
supervised_loss = losses.softmax_cross_entropy(noisy_logits, y)
noisy_preds = tf.nn.softmax(noisy_logits)
# Now run the clean data through the clean encoder
encoder = X
for layer_number, n_hidden_units in enumerate(hidden_units):
encoder = fully_connected_encoder(
encoder,
n_hidden_units,
activation=activation,
noise_type=None,
random_state=random_state,
reuse=True, # re-use variables from previous encoder
hidden_outputs_collections=CLEAN_ENCODER_HIDDEN_STATE,
outputs_collections=CLEAN_ENCODER_OUTPUTS,
scope='encoding_layer_{:d}'.format(layer_number))
# clean classification head
clean_logits = fully_connected_encoder(
encoder,
n_hidden_units=n_classes,
activation='identity',
noise_type=None,
random_state=random_state,
reuse=True,
hidden_outputs_collections=CLEAN_ENCODER_HIDDEN_STATE,
outputs_collections=CLEAN_ENCODER_OUTPUTS,
scope='encoding_layer_{:d}'.format(n_layers))
predictions = tf.nn.softmax(clean_logits)
# go back through the decoder
reconstruction_costs = []
decoder = noisy_preds
for layer_number in xrange(n_layers, -1, -1):
hidden_encoder = lookup_encoder_hidden_state(layer_number,
encoder_type='noisy')
use_linear_transform = False if layer_number == n_layers else True
decoder = fully_connected_decoder(
decoder,
hidden_encoder,
linear_transformation=use_linear_transform,
scope='decoding_layer_{:d}'.format(layer_number))
# calculate reconstruction loss
encoder_mean, encoder_variance = lookup_encoder_batch_statistics(
layer_number, encoder_type='clean')
z_est_bn = (decoder - encoder_mean) / encoder_variance
# calculate unsupervised loss
clean_hidden_encoder = lookup_encoder_hidden_state(
layer_number, encoder_type='clean')
loss = tf.reduce_sum((z_est_bn - clean_hidden_encoder)**2, 1)
unsupervised_loss = tf.reduce_mean(loss)
reconstruction_costs.append(unsupervised_loss)
total_reconstruction_loss = tf.add_n(reconstruction_costs)
total_loss = supervised_loss + 0.10 * total_reconstruction_loss
return predictions, total_loss