def _spatial_transformer(self, name, x, in_filters, arr_out_filters):
width = x.get_shape().as_list()[1]
height = x.get_shape().as_list()[2]
with tf.variable_scope(name):
_x = MaxPooling2D(x, name='pool1')
with tf.variable_scope('conv_1'):
_x = Conv2D(_x,
in_filters,
5,
arr_out_filters[0],
name='conv_1')
_x = BatchNormalization(_x,
self.mode == 'train',
name='batch1')
_x = MaxPooling2D(_x, use_relu=True, name='pool2')
with tf.variable_scope('conv_2'):
_x = Conv2D(_x,
arr_out_filters[0],
5,
arr_out_filters[1],
name='conv_2')
_x = BatchNormalization(_x,
self.mode == 'train',
name='batch2')
_x = MaxPooling2D(_x, use_relu=True, name='pool3')
with tf.variable_scope('fc1'):
_x_flat, _x_size = Flatten(_x)
W_fc_loc1 = weight_variable([_x_size, arr_out_filters[2]])
b_fc_loc1 = bias_variable([arr_out_filters[2]])
h_fc_loc1 = tf.nn.tanh(
tf.matmul(_x_flat, W_fc_loc1) + b_fc_loc1)
h_fc_loc1 = slim.dropout(h_fc_loc1,
self._dropout,
is_training=(self.mode == 'train'
and self._dropout > 0),
scope='dropout')
with tf.variable_scope('fc2'):
W_fc_loc2 = weight_variable([arr_out_filters[2], 6])
# Use identity transformation as starting point
initial = np.array([[1., 0, 0], [0, 1., 0]])
initial = initial.astype('float32')
initial = initial.flatten()
b_fc_loc2 = tf.Variable(initial_value=initial,
name='b_fc_loc2')
h_fc_loc2 = tf.nn.tanh(
tf.matmul(h_fc_loc1, W_fc_loc2) + b_fc_loc2)
# %% We'll create a spatial transformer module to identify discriminative
# %% patches
out_size = (width, height)
h_trans = transformer(x, h_fc_loc2, out_size)
h_trans = tf.reshape(
h_trans, [self.hps.batch_size, width, height, in_filters])
return h_trans
def _spatial_transformer(self, name, x, in_filters, arr_out_filters):
width = x.get_shape().as_list()[1]
height = x.get_shape().as_list()[2]
with tf.variable_scope(name):
W_fc_loc1 = weight_variable([width * height * in_filters, arr_out_filters[2]])
b_fc_loc1 = bias_variable([arr_out_filters[2]])
W_fc_loc2 = weight_variable([arr_out_filters[2], 6])
# Use identity transformation as starting point
initial = np.array([[1., 0, 0], [0, 1., 0]])
initial = initial.astype('float32')
initial = initial.flatten()
b_fc_loc2 = tf.Variable(initial_value=initial, name='b_fc_loc2')
x_reshape = tf.reshape(x, [self.hps.batch_size, -1])
# %% Define the two layer localisation network
h_fc_loc1 = tf.nn.tanh(tf.matmul(x_reshape, W_fc_loc1) + b_fc_loc1)
h_fc_loc1 = self._batch_norm2(name, h_fc_loc1)
h_fc_loc1 = self._relu(h_fc_loc1)
h_fc_loc1 = slim.dropout(h_fc_loc1, self._dropout,
is_training=(self.mode == 'train'
and self._dropout
and self._dropout > 0),
scope='dropout')
h_fc_loc2 = tf.nn.tanh(tf.matmul(h_fc_loc1, W_fc_loc2) + b_fc_loc2)
# %% We'll create a spatial transformer module to identify discriminative
# %% patches
out_size = (width, height)
h_trans = transformer(x, h_fc_loc2, out_size)
h_trans = tf.reshape(h_trans, [self.hps.batch_size, width, height, in_filters])
return h_trans
def nn_layer(input_tensor,
input_dim,
output_dim,
layer_name,
act=tf.nn.relu,
method="xavier"):
"""Reusable code for making a simple neural net layer.
It does a matrix multiply, bias add, and then uses relu to nonlinearize.
It also sets up name scoping so that the resultant graph is easy to read,
and adds a number of summary ops.
"""
# Adding a name scope ensures logical grouping of the layers in the graph.
with tf.name_scope(layer_name):
# This Variable will hold the state of the weights for the layer
with tf.name_scope('weights'):
weights = weight_variable([input_dim, output_dim],
method=method,
name=layer_name)
variable_summaries(weights, layer_name + '/weights')
with tf.name_scope('biases'):
biases = bias_variable([output_dim])
variable_summaries(biases, layer_name + '/biases')
with tf.name_scope('Wx_plus_b'):
preactivate = tf.matmul(input_tensor, weights) + biases
tf.histogram_summary(layer_name + '/pre_activations', preactivate)
if act is None:
activations = preactivate
else:
activations = act(preactivate, 'activation')
tf.histogram_summary(layer_name + '/activations', activations)
return activations
def inference(image, keep_prob):
print("setting up vgg initialized conv layers ...")
model_data = utils.get_model_data(FLAGS.model_path)
mean = model_data['normalization'][0][0][0]
mean_pixel = np.mean(mean, axis=(0, 1))
weights = np.squeeze(model_data['layers'])
processed_image = utils.process_image(image, mean_pixel)
with tf.variable_scope("inference"):
image_net = vgg_net(weights, processed_image)
conv_final_layer = image_net["conv5_3"]
pool5 = utils.max_pool_2x2(conv_final_layer, "pool5")
W6 = utils.weight_variable([7, 7, 512, 4096], name="W6")
b6 = utils.bias_variable([4096], name="b6")
conv6 = utils.conv2d_basic(pool5, W6, b6, name="conv6")
relu6 = tf.nn.relu(conv6, name="relu6")
if FLAGS.debug:
utils.add_activation_summary(relu6)
relu_dropout6 = tf.nn.dropout(relu6, keep_prob=keep_prob)
W7 = utils.weight_variable([1, 1, 4096, 4096], name="W7")
b7 = utils.weight_variable([4096], name="b7")
conv7 = utils.conv2d_basic(relu_dropout6, W7, b7, name="conv7")
relu7 = tf.nn.relu(conv7, name="relu7")
if FLAGS.debug:
utils.add_activation_summary(relu7)
relu_dropout7 = tf.nn.dropout(relu7, keep_prob=keep_prob)
W8 = utils.weight_variable([1, 1, 4096, NUM_OF_CLASSES], name="W8")
b8 = utils.bias_variable([NUM_OF_CLASSES], name="b8")
conv8 = utils.conv2d_basic(relu_dropout7, W8, b8, name="conv8")
# now to upscale to actual image size
deconv_shape1 = image_net["pool4"].get_shape()
W_t1 = utils.weight_variable([4, 4, deconv_shape1[3].value, NUM_OF_CLASSES], name="W_t1")
b_t1 = utils.bias_variable([deconv_shape1[3].value], name="b_t1")
conv_t1 = utils.conv2d_transpose_strided(conv8, W_t1, b_t1, "conv_t1", output_shape=tf.shape(image_net("pool4")))
fuse_1 = tf.add(conv_t1, image_net["pool4"], name="fuse_1")
deconv_shape2 = image_net["pool3"].get_shape()
W_t2 = utils.weight_variable([4, 4, deconv_shape2[3].value, deconv_shape1[3].value], name="W_t2")
b_t2 = utils.bias_variable([deconv_shape2[3].value], name="b_t2")
conv_t2 = utils.conv2d_transpose_strided(fuse_1, W_t2, b_t2, "conv_t2", output_shape=tf.shape(image_net("pool3")))
fuse_2 = tf.add(conv_t2, image_net["pool3"], name="fuse_2")
shape = tf.shape(image)
deconv_shape3 = tf.stack([shape[0], shape[1], shape[2], NUM_OF_CLASSES])
W_t3 = utils.weight_variable([16, 16, NUM_OF_CLASSES, deconv_shape2[3].value], name="W_t3")
b_t3 = utils.bias_variable([NUM_OF_CLASSES], name="b_t3")
conv_t3 = utils.conv2d_transpose_strided(fuse_2, W_t3, b_t3, "conv_t3", output_shape=deconv_shape3, stride=8)
annotation_pred = tf.argmax(conv_t3, axis=2, name="prediction")
return tf.expand_dims(annotation_pred, axi=3), conv_t3
def buildConv2D(inputWidth, inputHeight, inputConv, n_input_filters,filter_size, n_filters, stride, pad):
WConv = weight_variable([filter_size, filter_size, n_input_filters, n_filters])
bConv = bias_variable([n_filters])
hConv = tf.nn.relu(
tf.nn.conv2d(input=inputConv,
filter=WConv,
strides=stride,
padding=pad) +
bConv)
w,h = computeVolume(inputWidth, inputHeight, stride)
return hConv,w,h
def block_flow(x, layers, in_features, growth, is_training, keep_prob,
weights):
current = x
features = in_features
for idx in range(layers):
print(current.get_shape())
W = weight_variable([3, 3, features, growth])
weights.append(W)
b = bias_variable([growth])
tmp = batch_activ_conv(current, is_training, keep_prob, W, b)
current = tf.concat((current, tmp), axis=3)
features += growth
return current, features, weights
def spTrans(x_tensor,width, height, channels, n_loc,keep_prob):
resolution = width * height * channels
W_fc_loc1 = weight_variable([resolution, n_loc])
b_fc_loc1 = bias_variable([n_loc])
W_fc_loc2 = weight_variable([n_loc, 6])
# Use identity transformation as starting point
initial = np.array([[1., 0, 0], [0, 1., 0]])
initial = initial.astype('float32')
initial = initial.flatten()
b_fc_loc2 = tf.Variable(initial_value=initial, name='b_fc_loc2')
# Two layer localisation network
h_fc_loc1 = tf.nn.tanh(tf.matmul(x, W_fc_loc1) + b_fc_loc1)
# dropout (reduce overfittin)
h_fc_loc1_drop = tf.nn.dropout(h_fc_loc1, keep_prob)
# %% Second layer
h_fc_loc2 = tf.nn.tanh(tf.matmul(h_fc_loc1_drop, W_fc_loc2) + b_fc_loc2)
# spatial transformer
out_size = (width, height)
h_trans = transformer(x_tensor, h_fc_loc2, out_size)
return h_trans, b_fc_loc2, h_fc_loc2
def block_single(x1, x2, x3, layers, in_features, growth, is_training,
keep_prob, weights):
current1 = x1
current2 = x2
current3 = x3
features = in_features
for idx in range(layers):
print(current1.get_shape())
W = weight_variable([3, 3, features, growth])
weights.append(W)
b = bias_variable([growth])
tmp1 = batch_activ_conv(current1, is_training, keep_prob, W, b)
tmp2 = batch_activ_conv(current2, is_training, keep_prob, W, b)
tmp3 = batch_activ_conv(current3, is_training, keep_prob, W, b)
current1 = tf.concat((current1, tmp1), axis=3)
current2 = tf.concat((current2, tmp2), axis=3)
current3 = tf.concat((current3, tmp3), axis=3)
features += growth
return current1, current2, current3, features, weights
def buildSoftMax(inputSoftMax,n_fc,classes):
W_fc = weight_variable([n_fc, classes])
b_fc = bias_variable([classes])
y_logits = tf.matmul(inputSoftMax, W_fc) + b_fc
return y_logits
def buildFc(inputFc,height,width,n_filters,n_fc,keep_prob):
W_fc = weight_variable([height * width * n_filters, n_fc])
b_fc = bias_variable([n_fc])
h_fc = tf.nn.relu(tf.matmul(inputFc, W_fc) + b_fc)
h_fc_drop = tf.nn.dropout(h_fc, keep_prob)
return h_fc_drop
# %% Since x is currently [batch, height*width], we need to reshape to a # 4-D tensor to use it in a convolutional graph. If one component of # `shape` is the special value -1, the size of that dimension is # computed so that the total size remains constant. Since we haven't # defined the batch dimension's shape yet, we use -1 to denote this # dimension should not change size. x_tensor = tf.reshape(x, [-1, 100, 100, 1]) #%% localizaton network keep_prob = tf.placeholder(tf.float32) l_pool0_loc = tf.nn.max_pool(x_tensor,ksize=[1,2,2,1],strides=[1,2,2,1],padding='VALID') W_conv0_loc = weight_variable([3,3,1,20]) l_conv0_loc = tf.nn.relu(tf.nn.conv2d(l_pool0_loc,W_conv0_loc,strides=[1,1,1,1],padding='VALID')) l_pool1_loc = tf.nn.max_pool(l_conv0_loc,ksize=[1,2,2,1],strides =[1,2,2,1],padding='VALID') W_conv1_loc = weight_variable([3,3,20,20]) l_conv1_loc = tf.nn.relu(tf.nn.conv2d(l_pool1_loc,W_conv1_loc,strides=[1,1,1,1],padding='VALID')) l_pool2_loc = tf.nn.max_pool(l_conv1_loc,ksize=[1,2,2,1],strides =[1,2,2,1],padding='VALID') W_conv2_loc = weight_variable([3,3,20,20]) l_conv2_loc = tf.nn.relu(tf.nn.conv2d(l_pool2_loc,W_conv2_loc,strides=[1,1,1,1],padding='VALID') )
dw_h_convs_test = OrderedDict()
dw_h_convs1 = OrderedDict()
dw_h_convs1_test = OrderedDict()
convs2 = []
dw_h_convs2 = OrderedDict()
dw_h_convs2_test = OrderedDict()
convs3 = []
dw_h_convs3 = OrderedDict()
dw_h_convs3_test = OrderedDict()
convs_comb = []
dw_h_convs_comb = OrderedDict()
dw_h_convs_comb_test = OrderedDict()
#flow path
W = weight_variable([filter_size, filter_size, n_channels, 16])
b = bias_variable([16])
weights.append(W)
current_flow = conv2d_same(in_node, W, b)
current_flow, features, weights = block_flow(current_flow, layers_flow, 16,
12, is_training, keep_prob,
weights)
print("Flow after Block1:", current_flow.get_shape())
W = weight_variable([1, 1, features, features // 4])
b = bias_variable([features // 4])
weights.append(W)
current_flow = batch_activ_conv(current_flow, is_training, keep_prob, W, b)
#current_flow = avg_pool(current_flow, 2)
keep_prob = tf.placeholder(tf.float32)
# %% Second layer
ful_length = 200
h_fc_loc11 = nn_layer(x,
3200,
ful_length,
'fc_loc11',
act=tf.nn.tanh,
method='zeros')
h_fc_loc11_drop = tf.nn.dropout(h_fc_loc11, keep_prob)
with tf.name_scope('fc_loc12'):
with tf.name_scope('weights'):
W_fc_loc12 = weight_variable([ful_length, 6], method='zeros')
variable_summaries(W_fc_loc12, 'fc_loc12' + '/weights')
# Use identity transformation as starting point
with tf.name_scope('biases'):
initial = np.array([[1., 0, 0], [0, 1., 0]])
initial = initial.astype('float32')
initial = initial.flatten()
b_fc_loc12 = tf.Variable(initial_value=initial, name='b_fc_loc12')
variable_summaries(b_fc_loc12, 'fc_loc12' + '/biases')
with tf.name_scope('Wx_plus_b'):
fc_loc12_preactivate = tf.matmul(h_fc_loc11_drop,
W_fc_loc12) + b_fc_loc12
tf.histogram_summary('fc_loc12' + '/pre_activations',
fc_loc12_preactivate)
h_fc_loc12 = tf.nn.tanh(fc_loc12_preactivate, 'activation')
def build_fc_freq_4_30_NoTiedWeight_BN_Tiny(x,
x_dim,
keep_prob,
is_training,
gamma=1e-7,
activation='relu'):
# FC1
with tf.name_scope("FC1"):
fc1_dim = 500
W_fc1 = weight_variable([x_dim, fc1_dim])
b_fc1 = weight_variable([fc1_dim])
h_fc1 = tf.nn.elu(tf.matmul(x, W_fc1) + b_fc1)
h_fc1_bn = batch_norm_wrapper(h_fc1, is_training)
# dropout
#with tf.name_scope("Dropout1"):
# h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)
# FC2
with tf.name_scope("FC2"):
fc2_dim = 300
W_fc2 = weight_variable([fc1_dim, fc2_dim])
b_fc2 = weight_variable([fc2_dim])
h_fc2 = tf.nn.elu(tf.matmul(h_fc1_bn, W_fc2) + b_fc2)
h_fc2_bn = batch_norm_wrapper(h_fc2, is_training)
# dropout
#with tf.name_scope("Dropout2"):
# h_fc2_drop = tf.nn.dropout(h_fc2, keep_prob)
# FC3
with tf.name_scope("FC3"):
fc3_dim = 100
W_fc3 = weight_variable([fc2_dim, fc3_dim])
b_fc3 = weight_variable([fc3_dim])
h_fc3 = tf.nn.elu(tf.matmul(h_fc2_bn, W_fc3) + b_fc3)
h_fc3_bn = batch_norm_wrapper(h_fc3, is_training)
# FC4
with tf.name_scope("FCFeat"):
fc4_dim = 64
W_fc4 = weight_variable([fc3_dim, fc4_dim])
b_fc4 = weight_variable([fc4_dim])
h_fc4 = tf.nn.elu(tf.matmul(h_fc3_bn, W_fc4) + b_fc4, name="feature")
h_fc4_bn = batch_norm_wrapper(h_fc4, is_training)
# FC5
with tf.name_scope("FC5"):
fc5_dim = 300
W_fc5 = weight_variable([fc4_dim, fc5_dim])
b_fc5 = weight_variable([fc5_dim])
h_fc5 = tf.nn.elu(tf.matmul(h_fc4_bn, W_fc5) + b_fc5)
h_fc5_bn = batch_norm_wrapper(h_fc5, is_training)
with tf.name_scope("FC6"):
fc6_dim = 1000
W_fc6 = weight_variable([fc5_dim, fc6_dim])
b_fc6 = weight_variable([fc6_dim])
h_fc6 = tf.nn.elu(tf.matmul(h_fc5_bn, W_fc6) + b_fc6)
h_fc6_bn = batch_norm_wrapper(h_fc6, is_training)
with tf.name_scope("FC7"):
fc7_dim = 500
W_fc7 = weight_variable([fc6_dim, fc7_dim])
b_fc7 = weight_variable([fc7_dim])
h_fc7 = tf.nn.elu(tf.matmul(h_fc6_bn, W_fc7) + b_fc7)
h_fc7_bn = batch_norm_wrapper(h_fc7, is_training)
# dropout
#with tf.name_scope("Dropout7"):
# h_fc7_drop = tf.nn.dropout(h_fc7, keep_prob)
# FC8
with tf.name_scope("FC8"):
fc8_dim = x_dim
W_fc8 = weight_variable([fc7_dim, fc8_dim])
b_fc8 = weight_variable([fc8_dim])
h_fc8 = tf.matmul(h_fc7_bn, W_fc8) + b_fc8
# LOSS
with tf.name_scope("loss"):
y = h_fc8 + 1e-10
l2_loss = tf.sqrt(tf.reduce_mean(tf.square(y - x)))
#entropy_loss = - tf.reduce_mean(x * tf.log(y))
#loss = entropy_loss + l2_loss
#loss = l2_loss
l1_loss_sum = tf.reduce_sum(tf.abs(W_fc1)) + \
tf.reduce_sum(tf.abs(W_fc2)) + \
tf.reduce_sum(tf.abs(W_fc3)) + \
tf.reduce_sum(tf.abs(W_fc4)) + \
tf.reduce_sum(tf.abs(W_fc5)) + \
tf.reduce_sum(tf.abs(W_fc6)) + \
tf.reduce_sum(tf.abs(W_fc7)) + \
tf.reduce_sum(tf.abs(W_fc7)) + \
tf.reduce_sum(tf.abs(W_fc8))
l1_loss = l1_loss_sum * gamma
loss = l2_loss
tf.summary.scalar("loss", loss)
summary_op = tf.summary.merge_all()
return loss, y, l1_loss
# %% Placeholders for 40x40 resolution
x = tf.placeholder(tf.float32, [None, 1600])
y = tf.placeholder(tf.float32, [None, 10])
# %% Since x is currently [batch, height*width], we need to reshape to a
# 4-D tensor to use it in a convolutional graph. If one component of
# `shape` is the special value -1, the size of that dimension is
# computed so that the total size remains constant. Since we haven't
# defined the batch dimension's shape yet, we use -1 to denote this
# dimension should not change size.
x_tensor = tf.reshape(x, [-1, 40, 40, 1])
# %% We'll setup the two-layer localisation network to figure out the parameters for an affine transformation of the input
# %% Create variables for fully connected layer
W_fc_loc1 = weight_variable([1600, 20])
b_fc_loc1 = bias_variable([20])
W_fc_loc2 = weight_variable([20, 6])
initial = np.array([[1.,0, 0],[0,1.,0]]) # Use identity transformation as starting point
initial = initial.astype('float32')
initial = initial.flatten()
b_fc_loc2 = tf.Variable(initial_value=initial, name='b_fc_loc2')
# %% Define the two layer localisation network
h_fc_loc1 = tf.nn.tanh(tf.matmul(x, W_fc_loc1) + b_fc_loc1)
# %% We can add dropout for regularizing and to reduce overfitting like so:
keep_prob = tf.placeholder(tf.float32)
h_fc_loc1_drop = tf.nn.dropout(h_fc_loc1, keep_prob)
# %% Second layer
h_fc_loc2 = tf.nn.tanh(tf.matmul(h_fc_loc1_drop, W_fc_loc2) + b_fc_loc2)
def main():
"""
Runs a simple linear regression model on the mnist dataset.
"""
# Load the mnist dataset. Class stores the train, validation and testing sets as numpy arrays.
mnist = input_data.read_data_sets('MNIST_data', one_hot=True)
# Create a tensforlow session.
sess = tf.InteractiveSession()
# Create the computational graph. Start with creating placeholders for the input and output data.
# Input placeholder.
input_placeholder = tf.placeholder(tf.float32, shape=[None, 784])
# Output placeholder.
labeled_data = tf.placeholder(tf.float32, shape=[None, 10])
# Reshape input to a 4D tensor of [ -1 , width, height, channels]. -1 ensures the size remains consitent with
# the original size.
image_shape = [-1, 28, 28, 1]
input_image = tf.reshape(input_placeholder, image_shape)
# Create convolutional layers containing 2 convolutional layers and 1 fully connected layer.
# Layer 1 computes 32 features for each 5x5 patch.
conv1_weights = tf_utils.weight_variable([5, 5, 1, 32])
conv1_bias = tf_utils.bias_variable([32])
# Apply ReLU activation and max pool.
conv1_act = tf.nn.relu(
tf_utils.conv2d(input_image, conv1_weights) + conv1_bias)
conv1_pool = tf_utils.max_pool_2x2(conv1_act)
# Layer 2 computes 64 features of 5x5 patch.
conv2_weights = tf_utils.weight_variable([5, 5, 32, 64])
conv2_bias = tf_utils.bias_variable([64])
# Apply ReLU activation and max pool.
conv2_act = tf.nn.relu(
tf_utils.conv2d(conv1_pool, conv2_weights) + conv2_bias)
conv2_pool = tf_utils.max_pool_2x2(conv2_act)
# Add fully connected layers.
fc1_weights = tf_utils.weight_variable([7 * 7 * 64, 1024])
fc1_bias = tf_utils.bias_variable([1024])
# Apply Relu activation to flattened conv2d pool layer.
conv2_flat = tf.reshape(conv2_pool, [-1, 7 * 7 * 64])
fc1_act = tf.nn.relu(tf.matmul(conv2_flat, fc1_weights) + fc1_bias)
# Add dropout before the readout layer.
keep_prob = tf.placeholder(tf.float32)
dropout = tf.nn.dropout(fc1_act, keep_prob)
# Add the readout layer for the 10 classes.
readout_weights = tf_utils.weight_variable([1024, 10])
readout_bias = tf_utils.bias_variable([10])
readout_act = tf.matmul(dropout, readout_weights) + readout_bias
# Cross entropy loss between the output labels and the model.
cross_entropy = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=labeled_data,
logits=readout_act))
# Define the training step with a learning rate for gradient descent and our cross entropy loss.
learning_rate = 1e-4
train_step = tf.train.AdamOptimizer(learning_rate).minimize(cross_entropy)
# Initialize all variables.
sess.run(tf.global_variables_initializer())
# Training model evaluation placeholders.
# Define a placeholder for comparing equality between output and labels.
predictions = tf.equal(tf.argmax(labeled_data, 1),
tf.argmax(readout_act, 1))
accuracy = tf.reduce_mean(tf.cast(predictions, tf.float32))
# Run the training for a n steps.
steps = 10000
batch_size = 50
for step in xrange(steps):
# Sample a batch from the mnist dataset.
batch = mnist.train.next_batch(batch_size)
# Create a dict of the data from the sampled batch and run one training step.
train_step.run(feed_dict={
input_placeholder: batch[0],
labeled_data: batch[1],
keep_prob: 0.5
})
# Print the training error after every 100 steps.
if step % 100 == 0:
train_accuracy = accuracy.eval(
feed_dict={
input_placeholder: batch[0],
labeled_data: batch[1],
keep_prob: 1.0
})
print "Step: ", step, " | Train Accuracy: ", train_accuracy
print "Accuracy: ", accuracy.eval(
feed_dict={
input_placeholder: mnist.test.images,
labeled_data: mnist.test.labels,
keep_prob: 1.0
})
u, [-1, inputs_train.shape[1], inputs_train.shape[2], n_channels])
in_node = u_img
in_node_test = u_img
weights = []
biases = []
convs = []
dw_h_convs = OrderedDict()
dw_h_convs_test = OrderedDict()
print("in_node:", in_node.get_shape())
for layer in range(0, layers):
stddev = np.sqrt(2 / (filter_size**2 * features))
if layer == 0:
w1 = weight_variable(
[filter_size, filter_size, n_channels, features], stddev)
else:
w1 = weight_variable(
[filter_size, filter_size, features_bottleneck, features],
stddev)
b1 = bias_variable([features])
w2 = weight_variable([1, 1, features, features_bottleneck], stddev)
b2 = bias_variable([features_bottleneck])
conv1 = conv2d(in_node, w1, keep_prob)
print("layer:", layer, "conv1:", conv1.get_shape())
conv1_bn = tf.layers.batch_normalization(inputs=conv1,
axis=-1,
momentum=0.9,
#weights.append(w1)
#biases.append(b1)
convs.append(conv1)
if 0 < layer < layers - 1:
in_node = tf.concat((in_node, dw_h_convs[layer]), axis=-1)
elif layer == 0:
in_node = dw_h_convs[layer]
print(in_node.get_shape())
#in_node = dw_h_convs[layers-1]
print(in_node.get_shape()[3])
# Output Map
stddev = np.sqrt(2 / (filter_size**2 * (in_node.get_shape()[3].value)))
weight = weight_variable([1, 1, (in_node.get_shape()[3].value), 1], stddev)
bias = bias_variable([1], -0.5)
conv = conv2d(in_node, weight, tf.constant(1.0))
pred_mask = tf.nn.sigmoid(conv + bias)
#pred_mask = tf.nn.relu(conv + bias)
# loss
#loss = tf.losses.mean_squared_error(pred_mask, crop(s_img,pred_mask))
#loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=s_, labels=s))
#loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=s_, labels=s))
loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(logits=conv + bias,
labels=s_img)) #sigmoid
#loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=pred_mask, labels=s_img)) #relu
param = 12 # Create variables for fully connected layer for the localisation network # def weight_variable(shape): # initial = tf.random_uniform(shape, minval=0, maxval = 0.1) # # initial = tf.random_normal(shape, mean= 0, stddev= 0.1) # # initial = tf.random_uniform(shape, minval=-0.1, maxval = 0.1) * tf.sqrt(1.0/shape[0]) # return tf.Variable(initial) # # def bias_variable(shape): # initial = tf.random_uniform(shape, minval=0, maxval = 0.1) # # initial = tf.random_normal(shape, mean= 0, stddev= 0.1) # # initial = tf.random_uniform(shape, minval=-0.1, maxval = 0.1) * tf.sqrt(1.0/shape[0]) # return tf.Variable(initial) W_fc_loc1 = weight_variable([mri_height * mri_width * mri_channel, 50]) b_fc_loc1 = bias_variable([50]) W_fc_loc2 = weight_variable([50, param]) b_fc_loc2 = tf.Variable(initial_value=initial, name='b_fc_loc2') keep_prob = tf.placeholder(tf.float32) x_flat = tf.layers.flatten(x) h_fc_loc1 = tf.nn.tanh(tf.matmul(x_flat, W_fc_loc1) + b_fc_loc1) h_fc_loc1_drop = tf.nn.dropout(h_fc_loc1, keep_prob) h_fc_loc2 = tf.nn.tanh(tf.matmul(h_fc_loc1_drop, W_fc_loc2) + b_fc_loc2) h_fc_loc2_mat = tf.reshape(h_fc_loc2, (-1, 3, 4)) h_trans = spatial_transformer_network(x, h_fc_loc2)
x = tf.placeholder(tf.float32, [None, resolution])
y = tf.placeholder(tf.float32, [None, numIndiv])
# %% Since x is currently [batch, height*width], we need to reshape to a
# 4-D tensor to use it in a convolutional graph. If one component of
# `shape` is the special value -1, the size of that dimension is
# computed so that the total size remains constant. Since we haven't
# defined the batch dimension's shape yet, we use -1 to denote this
# dimension should not change size.
x_tensor = tf.reshape(x, [-1, imsize[1], imsize[2], imsize[0]])
# %% We'll setup the two-layer localisation network to figure out the
# %% parameters for an affine transformation of the input
# %% Create variables for fully connected layer
W_fc_loc1 = weight_variable([resolution, 20])
b_fc_loc1 = bias_variable([20])
# 6 is the number of parameters needed for the affine transformer
W_fc_loc2 = weight_variable([20, 6])
# Use identity transformation as starting point
# the multiplication (2x2) matrix is set to be the identity and the
# sum (translation) is [0,0]
initial = np.array([[1., 0, 0], [0, 1., 0]])
initial = initial.astype('float32')
initial = initial.flatten()
b_fc_loc2 = tf.Variable(initial_value=initial, name='b_fc_loc2')
# %% Define the two layer localisation network
h_fc_loc1 = tf.nn.tanh(tf.matmul(x, W_fc_loc1) + b_fc_loc1)
# %% We can add dropout for regularizing and to reduce overfitting like so:
keep_prob = tf.placeholder(tf.float32)
dropout_rate = tf.placeholder(tf.float32)
######################################### VGG16 NETWORK NO WEIGHTS #########################################
with tf.device('/gpu:0'):
with tf.variable_scope('network'):
######################################### LOCALISATION NETWORK #########################################
x_loc_c1 = conv(32, (3, 3), activation='relu', padding='same', name='loc_conv1')(x_low)
x_loc_p1 = pool((5, 5), strides=(5, 5), name='loc_pool1')(x_loc_c1)
x_loc_c2 = conv(32, (3, 3), activation='relu', padding='same', name='loc_conv2')(x_loc_p1)
x_loc_p2 = pool((5, 5), strides=(5, 5), name='loc_pool2')(x_loc_c2)
x_loc_f1 = flatten(name='loc_flatten1')(x_loc_p2)
x_loc_de1 = dense(64, activation='relu', name='loc_fc1')(x_loc_f1)
W_fc_loc2 = weight_variable([64, 6])
initial = np.array([[1., 0, 0], [0, 1., 0]])
initial = initial.astype('float32')
initial = initial.flatten()
b_fc_loc2 = tf.Variable(initial_value=initial, name='b_fc_loc2')
h_fc_loc2 = tf.nn.tanh(tf.matmul(x_loc_de1, W_fc_loc2) + b_fc_loc2)
######################################### SPATIAL TRANSFORMER LAYER #########################################
o_size = (out_size, out_size)
h_trans = transformer(x, h_fc_loc2, o_size)
h_trans.set_shape([None, out_size, out_size, n_channels])
######################################### PRE-TRAINED NETWORK #########################################
PT_layer = PT_model(h_trans)
x_gap = glob_avg_pool(name='glob_avg_pool')(PT_layer)
in_node = u_img
weights = []
biases = []
convs = []
pools = OrderedDict()
deconv = OrderedDict()
dw_h_convs = OrderedDict()
up_h_convs = OrderedDict()
# down layers
for layer in range(0, layers):
features = 2**layer * features_root
stddev = np.sqrt(2 / (filter_size**2 * features))
if layer == 0:
w1 = weight_variable(
[filter_size, filter_size, n_channels, features], stddev)
else:
w1 = weight_variable(
[filter_size, filter_size, features // 2, features], stddev)
w2 = weight_variable([filter_size, filter_size, features, features],
stddev)
b1 = bias_variable([features])
b2 = bias_variable([features])
conv1 = conv2d(in_node, w1, keep_prob)
tmp_h_conv = tf.nn.relu(conv1 + b1)
conv2 = conv2d(tmp_h_conv, w2, keep_prob)
dw_h_convs[layer] = tf.nn.relu(conv2 + b2)
weights.append((w1, w2))
def GEDDnet(face,
left_eye,
right_eye,
keep_prob,
is_train,
subj_id,
vgg_path,
num_subj=15,
rf=[[2, 2], [3, 3], [5, 5], [11, 11]],
num_face=[64, 128, 64, 64, 128, 256, 64],
r=[[2, 2], [3, 3], [4, 5], [5, 11]],
num_eye=[64, 128, 64, 64, 128, 256],
num_comb=[0, 256]):
num_comb[0] = num_face[-1] + 2 * num_eye[-1]
vgg = np.load(vgg_path)
with tf.variable_scope("transfer"):
W_conv1_1 = tf.Variable(vgg['conv1_1_W'])
b_conv1_1 = tf.Variable(vgg['conv1_1_b'])
W_conv1_2 = tf.Variable(vgg['conv1_2_W'])
b_conv1_2 = tf.Variable(vgg['conv1_2_b'])
W_conv2_1 = tf.Variable(vgg['conv2_1_W'])
b_conv2_1 = tf.Variable(vgg['conv2_1_b'])
W_conv2_2 = tf.Variable(vgg['conv2_2_W'])
b_conv2_2 = tf.Variable(vgg['conv2_2_b'])
del vgg
""" define network """
# face
face_h_conv1_1 = tf.nn.relu(conv2d(face, W_conv1_1) + b_conv1_1)
face_h_conv1_2 = tf.nn.relu(conv2d(face_h_conv1_1, W_conv1_2) + b_conv1_2)
face_h_pool1 = max_pool_2x2(face_h_conv1_2)
face_h_conv2_1 = tf.nn.relu(conv2d(face_h_pool1, W_conv2_1) + b_conv2_1)
face_h_conv2_2 = tf.nn.relu(conv2d(face_h_conv2_1, W_conv2_2) +
b_conv2_2) / 100.
with tf.variable_scope("face"):
face_W_conv2_3 = weight_variable([1, 1, num_face[1], num_face[2]],
std=0.125)
face_b_conv2_3 = bias_variable([num_face[2]], std=0.001)
face_W_conv3_1 = weight_variable([3, 3, num_face[2], num_face[3]],
std=0.06)
face_b_conv3_1 = bias_variable([num_face[3]], std=0.001)
face_W_conv3_2 = weight_variable([3, 3, num_face[3], num_face[3]],
std=0.06)
face_b_conv3_2 = bias_variable([num_face[3]], std=0.001)
face_W_conv4_1 = weight_variable([3, 3, num_face[3], num_face[4]],
std=0.08)
face_b_conv4_1 = bias_variable([num_face[4]], std=0.001)
face_W_conv4_2 = weight_variable([3, 3, num_face[4], num_face[4]],
std=0.07)
face_b_conv4_2 = bias_variable([num_face[4]], std=0.001)
face_W_fc1 = weight_variable([6 * 6 * num_face[4], num_face[5]],
std=0.035)
face_b_fc1 = bias_variable([num_face[5]], std=0.001)
face_W_fc2 = weight_variable([num_face[5], num_face[6]], std=0.1)
face_b_fc2 = bias_variable([num_face[6]], std=0.001)
face_h_conv2_3 = tf.nn.relu(
conv2d(face_h_conv2_2, face_W_conv2_3) + face_b_conv2_3)
face_h_conv2_3_norm = tf.layers.batch_normalization(face_h_conv2_3,
training=is_train,
scale=False,
renorm=True,
name="f_conv2_3")
face_h_conv3_1 = tf.nn.relu(
dilated2d(face_h_conv2_3_norm, face_W_conv3_1, rf[0]) +
face_b_conv3_1)
face_h_conv3_1_norm = tf.layers.batch_normalization(face_h_conv3_1,
training=is_train,
scale=False,
renorm=True,
name="f_conv3_1")
face_h_conv3_2 = tf.nn.relu(
dilated2d(face_h_conv3_1_norm, face_W_conv3_2, rf[1]) +
face_b_conv3_2)
face_h_conv3_2_norm = tf.layers.batch_normalization(face_h_conv3_2,
training=is_train,
scale=False,
renorm=True,
name="f_conv3_2")
face_h_conv4_1 = tf.nn.relu(
dilated2d(face_h_conv3_2_norm, face_W_conv4_1, rf[2]) +
face_b_conv4_1)
face_h_conv4_1_norm = tf.layers.batch_normalization(face_h_conv4_1,
training=is_train,
scale=False,
renorm=True,
name="f_conv4_1")
face_h_conv4_2 = tf.nn.relu(
dilated2d(face_h_conv4_1_norm, face_W_conv4_2, rf[3]) +
face_b_conv4_2)
face_h_conv4_2_norm = tf.layers.batch_normalization(face_h_conv4_2,
training=is_train,
scale=False,
renorm=True,
name="f_conv4_2")
face_h_pool4_flat = tf.reshape(face_h_conv4_2_norm,
[-1, 6 * 6 * num_face[4]])
face_h_fc1 = tf.nn.relu(
tf.matmul(face_h_pool4_flat, face_W_fc1) + face_b_fc1)
face_h_fc1_norm = tf.layers.batch_normalization(face_h_fc1,
training=is_train,
scale=False,
renorm=True,
name="f_fc1")
face_h_fc1_drop = tf.nn.dropout(face_h_fc1_norm, keep_prob)
face_h_fc2 = tf.nn.relu(
tf.matmul(face_h_fc1_drop, face_W_fc2) + face_b_fc2)
face_h_fc2_norm = tf.layers.batch_normalization(face_h_fc2,
training=is_train,
scale=False,
renorm=True,
name="f_fc2")
eye1_h_conv1_1 = tf.nn.relu(conv2d(left_eye, W_conv1_1) + b_conv1_1)
eye1_h_conv1_2 = tf.nn.relu(conv2d(eye1_h_conv1_1, W_conv1_2) + b_conv1_2)
eye1_h_pool1 = max_pool_2x2(eye1_h_conv1_2)
eye1_h_conv2_1 = tf.nn.relu(conv2d(eye1_h_pool1, W_conv2_1) + b_conv2_1)
eye1_h_conv2_2 = tf.nn.relu(conv2d(eye1_h_conv2_1, W_conv2_2) +
b_conv2_2) / 100.
eye2_h_conv1_1 = tf.nn.relu(conv2d(right_eye, W_conv1_1) + b_conv1_1)
eye2_h_conv1_2 = tf.nn.relu(conv2d(eye2_h_conv1_1, W_conv1_2) + b_conv1_2)
eye2_h_pool1 = max_pool_2x2(eye2_h_conv1_2)
eye2_h_conv2_1 = tf.nn.relu(conv2d(eye2_h_pool1, W_conv2_1) + b_conv2_1)
eye2_h_conv2_2 = tf.nn.relu(conv2d(eye2_h_conv2_1, W_conv2_2) +
b_conv2_2) / 100.
with tf.variable_scope("eye"):
# left eye
eye_W_conv2_3 = weight_variable([1, 1, num_eye[1], num_eye[2]],
std=0.125)
eye_b_conv2_3 = bias_variable([num_eye[2]], std=0.001)
eye_W_conv3_1 = weight_variable([3, 3, num_eye[2], num_eye[3]],
std=0.06)
eye_b_conv3_1 = bias_variable([num_eye[3]], std=0.001)
eye_W_conv3_2 = weight_variable([3, 3, num_eye[3], num_eye[3]],
std=0.06)
eye_b_conv3_2 = bias_variable([num_eye[3]], std=0.001)
eye_W_conv4_1 = weight_variable([3, 3, num_eye[3], num_eye[4]],
std=0.06)
eye_b_conv4_1 = bias_variable([num_eye[4]], std=0.001)
eye_W_conv4_2 = weight_variable([3, 3, num_eye[4], num_eye[4]],
std=0.04)
eye_b_conv4_2 = bias_variable([num_eye[4]], std=0.001)
eye1_W_fc1 = weight_variable([4 * 6 * num_eye[4], num_eye[5]],
std=0.026)
eye1_b_fc1 = bias_variable([num_eye[5]], std=0.001)
eye2_W_fc1 = weight_variable([4 * 6 * num_eye[4], num_eye[5]],
std=0.026)
eye2_b_fc1 = bias_variable([num_eye[5]], std=0.001)
eye1_h_conv2_3 = tf.nn.relu(
conv2d(eye1_h_conv2_2, eye_W_conv2_3) + eye_b_conv2_3)
eye1_h_conv2_3_norm = tf.layers.batch_normalization(eye1_h_conv2_3,
training=is_train,
scale=False,
renorm=True,
name="e_conv2_3")
eye1_h_conv3_1 = tf.nn.relu(
dilated2d(eye1_h_conv2_3_norm, eye_W_conv3_1, r[0]) +
eye_b_conv3_1)
eye1_h_conv3_1_norm = tf.layers.batch_normalization(eye1_h_conv3_1,
training=is_train,
scale=False,
renorm=True,
name="e_conv3_1")
eye1_h_conv3_2 = tf.nn.relu(
dilated2d(eye1_h_conv3_1_norm, eye_W_conv3_2, r[1]) +
eye_b_conv3_2)
eye1_h_conv3_2_norm = tf.layers.batch_normalization(eye1_h_conv3_2,
training=is_train,
scale=False,
renorm=True,
name="e_conv3_2")
eye1_h_conv4_1 = tf.nn.relu(
dilated2d(eye1_h_conv3_2_norm, eye_W_conv4_1, r[2]) +
eye_b_conv4_1)
eye1_h_conv4_1_norm = tf.layers.batch_normalization(eye1_h_conv4_1,
training=is_train,
scale=False,
renorm=True,
name="e_conv4_1")
eye1_h_conv4_2 = tf.nn.relu(
dilated2d(eye1_h_conv4_1_norm, eye_W_conv4_2, r[3]) +
eye_b_conv4_2)
eye1_h_conv4_2_norm = tf.layers.batch_normalization(eye1_h_conv4_2,
training=is_train,
scale=False,
renorm=True,
name="e_conv4_2")
eye1_h_pool4_flat = tf.reshape(eye1_h_conv4_2_norm,
[-1, 4 * 6 * num_eye[4]])
eye1_h_fc1 = tf.nn.relu(
tf.matmul(eye1_h_pool4_flat, eye1_W_fc1) + eye1_b_fc1)
eye1_h_fc1_norm = tf.layers.batch_normalization(eye1_h_fc1,
training=is_train,
scale=False,
renorm=True,
name="e1_fc1")
# right eye
eye2_h_conv2_3 = tf.nn.relu(
conv2d(eye2_h_conv2_2, eye_W_conv2_3) + eye_b_conv2_3)
eye2_h_conv2_3_norm = tf.layers.batch_normalization(eye2_h_conv2_3,
training=is_train,
scale=False,
renorm=True,
name="e_conv2_3",
reuse=True)
eye2_h_conv3_1 = tf.nn.relu(
dilated2d(eye2_h_conv2_3_norm, eye_W_conv3_1, r[0]) +
eye_b_conv3_1)
eye2_h_conv3_1_norm = tf.layers.batch_normalization(eye2_h_conv3_1,
training=is_train,
scale=False,
renorm=True,
name="e_conv3_1",
reuse=True)
eye2_h_conv3_2 = tf.nn.relu(
dilated2d(eye2_h_conv3_1_norm, eye_W_conv3_2, r[1]) +
eye_b_conv3_2)
eye2_h_conv3_2_norm = tf.layers.batch_normalization(eye2_h_conv3_2,
training=is_train,
scale=False,
renorm=True,
name="e_conv3_2",
reuse=True)
eye2_h_conv4_1 = tf.nn.relu(
dilated2d(eye2_h_conv3_2_norm, eye_W_conv4_1, r[2]) +
eye_b_conv4_1)
eye2_h_conv4_1_norm = tf.layers.batch_normalization(eye2_h_conv4_1,
training=is_train,
scale=False,
renorm=True,
name="e_conv4_1",
reuse=True)
eye2_h_conv4_2 = tf.nn.relu(
dilated2d(eye2_h_conv4_1_norm, eye_W_conv4_2, r[3]) +
eye_b_conv4_2)
eye2_h_conv4_2_norm = tf.layers.batch_normalization(eye2_h_conv4_2,
training=is_train,
scale=False,
renorm=True,
name="e_conv4_2",
reuse=True)
eye2_h_pool4_flat = tf.reshape(eye2_h_conv4_2_norm,
[-1, 4 * 6 * num_eye[4]])
eye2_h_fc1 = tf.nn.relu(
tf.matmul(eye2_h_pool4_flat, eye2_W_fc1) + eye2_b_fc1)
eye2_h_fc1_norm = tf.layers.batch_normalization(eye2_h_fc1,
training=is_train,
scale=False,
renorm=True,
name="e2_fc1")
# combine both eyes and face
with tf.variable_scope("combine"):
cls1_W_fc2 = weight_variable([num_comb[0], num_comb[1]], std=0.07)
cls1_b_fc2 = bias_variable([num_comb[1]], std=0.001)
cls1_W_fc3 = weight_variable([num_comb[1], 2], std=0.125)
cls1_b_fc3 = bias_variable([2], std=0.001)
cls1_h_fc1_norm = tf.concat(
[face_h_fc2_norm, eye1_h_fc1_norm, eye2_h_fc1_norm], axis=1)
cls1_h_fc1_drop = tf.nn.dropout(cls1_h_fc1_norm, keep_prob)
cls1_h_fc2 = tf.nn.relu(
tf.matmul(cls1_h_fc1_drop, cls1_W_fc2) + cls1_b_fc2)
cls1_h_fc2_norm = tf.layers.batch_normalization(cls1_h_fc2,
training=is_train,
scale=False,
renorm=True,
name="c_fc2")
cls1_h_fc2_drop = tf.nn.dropout(cls1_h_fc2_norm, keep_prob)
t_hat = tf.matmul(cls1_h_fc2_drop, cls1_W_fc3) + cls1_b_fc3
""" bias learning from subject id """
num_bias = (2 * num_subj, )
with tf.variable_scope("bias"):
bias_W_fc = weight_variable([num_bias[0], 2], std=0.125)
b_hat = tf.matmul(subj_id, bias_W_fc)
g_hat = t_hat + b_hat
l2_loss = (1e-2 * tf.nn.l2_loss(W_conv1_1) +
1e-2 * tf.nn.l2_loss(W_conv1_2) +
1e-2 * tf.nn.l2_loss(W_conv2_1) +
1e-2 * tf.nn.l2_loss(W_conv2_2) +
tf.nn.l2_loss(face_W_conv2_3) + tf.nn.l2_loss(face_W_conv3_1) +
tf.nn.l2_loss(face_W_conv3_2) + tf.nn.l2_loss(face_W_conv4_1) +
tf.nn.l2_loss(face_W_conv4_2) + tf.nn.l2_loss(face_W_fc1) +
tf.nn.l2_loss(face_W_fc2) + tf.nn.l2_loss(eye_W_conv2_3) +
tf.nn.l2_loss(eye_W_conv3_1) + tf.nn.l2_loss(eye_W_conv3_2) +
tf.nn.l2_loss(eye_W_conv4_1) + tf.nn.l2_loss(eye_W_conv4_2) +
tf.nn.l2_loss(eye1_W_fc1) + tf.nn.l2_loss(eye2_W_fc1) +
tf.nn.l2_loss(cls1_W_fc2) + tf.nn.l2_loss(cls1_W_fc3))
return g_hat, t_hat, bias_W_fc, l2_loss
# %% Since x is currently [batch, height*width], we need to reshape to a
# 4-D tensor to use it in a convolutional graph. If one component of
# `shape` is the special value -1, the size of that dimension is
# computed so that the total size remains constant. Since we haven't
# defined the batch dimension's shape yet, we use -1 to denote this
# dimension should not change size.
x_tensor = tf.reshape(x, [-1, 16, 16, 16, 1])
filter_size = 3
keep_prob = tf.placeholder(tf.float32)
# %% We'll setup the there-layer localisation network to figure out the
# %% parameters for an rotation transformation of the input
f1 = 8
w1 = weight_variable([filter_size, filter_size, filter_size, 1, f1])
b1 = bias_variable([f1])
h1 = tf.nn.relu(
tf.nn.conv3d(
input=x_tensor, filter=w1, strides=[1, 2, 2, 2, 1], padding='SAME') +
b1)
f2 = 4
w2 = weight_variable([filter_size, filter_size, filter_size, f1, f2])
b2 = bias_variable([f2])
h2 = tf.nn.relu(
tf.nn.conv3d(input=h1, filter=w2, strides=[1, 2, 2, 2, 1], padding='SAME')
+ b2)
h2 = tf.reshape(h2, [-1, 256])
w3 = weight_variable([256, 4])
b3 = bias_variable([4])
def build_fc_freq_5_TiedWeight_Small(x,
x_dim,
keep_prob,
is_training,
gamma=1e-7,
activation='relu'):
# FC1
with tf.name_scope("FC1"):
fc1_dim = 1024
W_fc1 = weight_variable([x_dim, fc1_dim])
b_fc1 = weight_variable([fc1_dim])
h_fc1 = tf.nn.elu(tf.matmul(x, W_fc1) + b_fc1)
h_fc1_bn = batch_norm_wrapper(h_fc1, is_training)
# dropout
#with tf.name_scope("Dropout1"):
# h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)
# FC2
with tf.name_scope("FC2"):
#fc2_dim = 4096
fc2_dim = 512
W_fc2 = weight_variable([fc1_dim, fc2_dim])
b_fc2 = weight_variable([fc2_dim])
h_fc2 = tf.nn.elu(tf.matmul(h_fc1_bn, W_fc2) + b_fc2)
h_fc2_bn = batch_norm_wrapper(h_fc2, is_training)
# dropout
#with tf.name_scope("Dropout2"):
# h_fc2_drop = tf.nn.dropout(h_fc2, keep_prob)
# FC3
with tf.name_scope("FC3"):
fc3_dim = 256
W_fc3 = weight_variable([fc2_dim, fc3_dim])
b_fc3 = weight_variable([fc3_dim])
h_fc3 = tf.nn.elu(tf.matmul(h_fc2_bn, W_fc3) + b_fc3)
h_fc3_bn = batch_norm_wrapper(h_fc3, is_training)
# FC4
with tf.name_scope("FCFeat"):
fc4_dim = 64
W_fc4 = weight_variable([fc3_dim, fc4_dim])
b_fc4 = weight_variable([fc4_dim])
h_fc4 = tf.nn.elu(tf.matmul(h_fc3_bn, W_fc4) + b_fc4, name="feature")
h_fc4_bn = batch_norm_wrapper(h_fc4, is_training)
# FC5
with tf.name_scope("FC5"):
fc5_dim = 256
W_fc5 = tf.transpose(W_fc4)
b_fc5 = weight_variable([fc5_dim])
h_fc5 = tf.nn.elu(tf.matmul(h_fc4_bn, W_fc5) + b_fc5)
h_fc5_bn = batch_norm_wrapper(h_fc5, is_training)
with tf.name_scope("FC6"):
fc6_dim = 512
W_fc6 = tf.transpose(W_fc3)
b_fc6 = weight_variable([fc6_dim])
h_fc6 = tf.nn.elu(tf.matmul(h_fc5_bn, W_fc6) + b_fc6)
h_fc6_bn = batch_norm_wrapper(h_fc6, is_training)
with tf.name_scope("FC7"):
fc7_dim = 1024
W_fc7 = tf.transpose(W_fc2)
b_fc7 = weight_variable([fc7_dim])
h_fc7 = tf.nn.elu(tf.matmul(h_fc6_bn, W_fc7) + b_fc7)
h_fc7_bn = batch_norm_wrapper(h_fc7, is_training)
# dropout
#with tf.name_scope("Dropout7"):
# h_fc7_drop = tf.nn.dropout(h_fc7, keep_prob)
# FC8
with tf.name_scope("FC8"):
fc8_dim = x_dim
#W_fc8 = weight_variable([fc7_dim, fc8_dim])
W_fc8 = tf.transpose(W_fc1)
b_fc8 = weight_variable([fc8_dim])
h_fc8 = tf.matmul(h_fc7_bn, W_fc8) + b_fc8
# LOSS
with tf.name_scope("loss"):
y = h_fc8 + 1e-10
l2_loss = tf.sqrt(tf.reduce_mean(tf.square(y - x)))
#entropy_loss = - tf.reduce_mean(x * tf.log(y))
#loss = entropy_loss + l2_loss
loss = l2_loss
l1_loss_sum = tf.reduce_sum(tf.abs(W_fc1)) + \
tf.reduce_sum(tf.abs(W_fc2)) + \
tf.reduce_sum(tf.abs(W_fc3)) + \
tf.reduce_sum(tf.abs(W_fc4))
# tf.reduce_sum(tf.abs(W_fc5)) + \
# tf.reduce_sum(tf.abs(W_fc6))
l1_loss = l1_loss_sum * gamma
#loss += l1_loss
tf_version = tf.__version__.rpartition('.')[0]
if parse_version(tf_version) >= parse_version('0.12.0'):
tf.summary.scalar("loss", loss)
else:
tf.scalar_summary("loss", loss)
# summary
if parse_version(tf_version) >= parse_version('0.12.0'):
summary_op = tf.summary.merge_all()
else:
summary_op = tf.merge_all_summaries()
return loss, y, l1_loss
# %% Placeholders for 40x40 resolution
x = tf.placeholder(tf.float32, [None, 1600])
y = tf.placeholder(tf.float32, [None, 10])
# %% Since x is currently [batch, height*width], we need to reshape to a
# 4-D tensor to use it in a convolutional graph. If one component of
# `shape` is the special value -1, the size of that dimension is
# computed so that the total size remains constant. Since we haven't
# defined the batch dimension's shape yet, we use -1 to denote this
# dimension should not change size.
x_tensor = tf.reshape(x, [-1, 40, 40, 1])
# %% We'll setup the two-layer localisation network to figure out the parameters for an affine transformation of the input
# %% Create variables for fully connected layer
W_fc_loc1 = weight_variable([1600, 20])
b_fc_loc1 = bias_variable([20])
W_fc_loc2 = weight_variable([20, 6])
initial = np.array([[1., 0, 0],
[0, 1.,
0]]) # Use identity transformation as starting point
initial = initial.astype('float32')
initial = initial.flatten()
b_fc_loc2 = tf.Variable(initial_value=initial, name='b_fc_loc2')
# %% Define the two layer localisation network
h_fc_loc1 = tf.nn.tanh(tf.matmul(x, W_fc_loc1) + b_fc_loc1)
# %% We can add dropout for regularizing and to reduce overfitting like so:
keep_prob = tf.placeholder(tf.float32)
h_fc_loc1_drop = tf.nn.dropout(h_fc_loc1, keep_prob)
# 4-D tensor to use it in a convolutional graph. If one component of
# `shape` is the special value -1, the size of that dimension is
# computed so that the total size remains constant. Since we haven't
# defined the batch dimension's shape yet, we use -1 to denote this
# dimension should not change size.
x_tensor = tf.reshape(x, [-1, 40, 40, 1])
x_tensor2 = tf.slice(x_tensor, [0, 10, 10, 0], [-1, 20, 20, -1])
with tf.name_scope('output'):
y = tf.placeholder(tf.float32, [None, 10])
with tf.name_scope('loc1'):
# %% We'll setup the two-layer localisation network to figure out the
# %% parameters for an affine transformation of the input
# %% Create variables for fully connected layer
W_fc_loc1 = weight_variable([1600, 20])
b_fc_loc1 = bias_variable([20])
# %% Define the two layer localisation network
h_fc_loc1 = tf.nn.tanh(tf.matmul(x, W_fc_loc1) + b_fc_loc1)
# %% We can add dropout for regularizing and to reduce overfitting like so:
keep_prob = tf.placeholder(tf.float32)
h_fc_loc1_drop = tf.nn.dropout(h_fc_loc1, keep_prob)
with tf.name_scope('loc2'):
W_fc_loc2 = weight_variable([20, 9])
# Use identity transformation as starting point
initial = np.array([[1., 0, 0], [0, 1., 0], [0, 0, 1.]])
initial = initial.astype('float32')
initial = initial.flatten()
b_fc_loc2 = tf.Variable(initial_value=initial, name='b_fc_loc2')
def build_model(resolution, numIndiv):
# %% Graph representation of our network
# %% Placeholders for imsize = height*width resolution and labels
x = tf.placeholder(tf.float32, [None, resolution])
y = tf.placeholder(tf.float32, [None, numIndiv])
# %% Since x is currently [batch, height*width], we need to reshape to a
# 4-D tensor to use it in a convolutional graph. If one component of
# `shape` is the special value -1, the size of that dimension is
# computed so that the total size remains constant. Since we haven't
# defined the batch dimension's shape yet, we use -1 to denote this
# dimension should not change size.
x_tensor = tf.reshape(x, [-1, imsize[1], imsize[2], imsize[0]])
# %% We'll setup the two-layer localisation network to figure out the
# %% parameters for an affine transformation of the input
# %% Create variables for fully connected layer
W_fc_loc1 = weight_variable([resolution, 20])
b_fc_loc1 = bias_variable([20])
# 6 is the number of parameters needed for the affine transformer
W_fc_loc2 = weight_variable([20, 6])
# Use identity transformation as starting point
# the multiplication (2x2) matrix is set to be the identity and the
# sum (translation) is [0,0]
initial = np.array([[1., 0, 0], [0, 1., 0]])
initial = initial.astype('float32')
initial = initial.flatten()
b_fc_loc2 = tf.Variable(initial_value=initial, name='b_fc_loc2')
# %% Define the two layer localisation network
h_fc_loc1 = tf.nn.tanh(tf.matmul(x, W_fc_loc1) + b_fc_loc1)
# %% We can add dropout for regularizing and to reduce overfitting like so:
keep_prob = tf.placeholder(tf.float32)
h_fc_loc1_drop = tf.nn.dropout(h_fc_loc1, keep_prob)
# %% Second layer
h_fc_loc2 = tf.nn.tanh(tf.matmul(h_fc_loc1_drop, W_fc_loc2) + b_fc_loc2)
# %% We'll create a spatial transformer module to identify discriminative
# %% patches
out_size = (imsize[1], imsize[2])
h_trans = transformer(x_tensor, h_fc_loc2, out_size)
# %% We'll setup the first convolutional layer
filter_size = 3
n_filters_1 = 16
# Weight matrix is [height x width x input_channels x output_channels]
W_conv1 = weight_variable([filter_size, filter_size, 1, n_filters_1])
# %% Bias is [output_channels]
b_conv1 = bias_variable([n_filters_1])
# %% Now we can build a graph which does the first layer of convolution:
# we define our stride as batch x height x width x channels
# instead of pooling, we use strides of 2 and more layers
# with smaller filters.
h_conv1 = tf.nn.relu(
tf.nn.conv2d(input=h_trans,
filter=W_conv1,
strides=[1, 2, 2, 1],
padding='SAME') +
b_conv1)
# %% And just like the first layer, add additional layers to create
# a deep net
n_filters_2 = 64
W_conv2 = weight_variable([filter_size, filter_size, n_filters_1, n_filters_2])
b_conv2 = bias_variable([n_filters_2])
h_conv2 = tf.nn.relu(
tf.nn.conv2d(input=h_conv1,
filter=W_conv2,
strides=[1, 2, 2, 1],
padding='SAME') +
b_conv2)
#
#
# # %% And why not a third...
n_filters_3 = 64
W_conv3 = weight_variable([filter_size, filter_size, n_filters_2, n_filters_3])
b_conv3 = bias_variable([n_filters_3])
h_conv3 = tf.nn.relu(
tf.nn.conv2d(input=h_conv2,
filter=W_conv3,
strides=[1, 2, 2, 1],
padding='SAME') +
b_conv3)
# # %% And why not a fourth...
# n_filters_4 = 16
# W_conv4 = weight_variable([filter_size, filter_size, n_filters_3, n_filters_4])
# b_conv4 = bias_variable([n_filters_4])
# h_conv4 = tf.nn.relu(
# tf.nn.conv2d(input=h_conv3,
# filter=W_conv4,
# strides=[1, 2, 2, 1],
# padding='SAME') +
# b_conv4)
# %% We'll now reshape so we can connect to a fully-connected layer:
# print resolution
lastVolSize = 12*12*n_filters_3
# lastVolSize = 7*7*n_filters_3
h_conv4_flat = tf.reshape(h_conv3, [-1, lastVolSize])
# %% Create a fully-connected layer:
n_fc = 4096
W_fc1 = weight_variable([lastVolSize, n_fc])
b_fc1 = bias_variable([n_fc])
h_fc1 = tf.nn.relu(tf.matmul(h_conv4_flat, W_fc1) + b_fc1)
h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)
# %% second fc
# n_fc2 = 256
# W_fc2 = weight_variable([n_fc, n_fc2])
# b_fc2 = bias_variable([n_fc2])
# h_fc2 = tf.nn.relu(tf.matmul(h_fc1, W_fc2) + b_fc2)
#
# h_fc2_drop = tf.nn.dropout(h_fc2, keep_prob)
# %% And finally our softmax layer:
W_fc3 = weight_variable([n_fc, numIndiv])
b_fc3 = bias_variable([numIndiv])
y_logits = tf.matmul(h_fc1_drop, W_fc3) + b_fc3
# %% Define loss/eval/training functions
cross_entropy = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=y_logits,labels=y))
opt = tf.train.AdamOptimizer(learning_rate=0.001, beta1=0.9, beta2=0.999, epsilon=1e-08, use_locking=False, name='Adam')
# opt = tf.train.FtrlOptimizer(0.01, learning_rate_power=-0.5, initial_accumulator_value=0.1, l1_regularization_strength=0.0, l2_regularization_strength=0.5, use_locking=False, name='Ftrl')
# opt = tf.train.RMSPropOptimizer(learning_rate=0.1, decay=0.16, momentum=0.9, epsilon=1.0, use_locking=False, name='RMSProp')
optimizer = opt.minimize(cross_entropy)
grads = opt.compute_gradients(cross_entropy, [b_fc_loc2])
# %% Monitor accuracy
correct_prediction = tf.equal(tf.argmax(y_logits, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, 'float'))
# Since x is currently [batch, height*width], we need to reshape to a
# 4-D tensor to use it in a convolutional graph. If one component of
# `shape` is the special value -1, the size of that dimension is
# computed so that the total size remains constant. Since we haven't
# defined the batch dimension's shape yet, we use -1 to denote this
# dimension should not change size.
x_tensor = tf.reshape(x, [-1, 40, 40, 1])
# We'll setup the two-layer localisation network to figure out the
# parameters for an affine transformation of the input
# Create variables for fully connected layer
numPixels = 1600
numNodesL1 = 20
numNodesL2 = 6
W_fc_loc1 = weight_variable([numPixels, numNodesL1])
b_fc_loc1 = bias_variable([numNodesL1])
W_fc_loc2 = weight_variable([numNodesL1, numNodesL2])
# Use identity transformation as starting point
initial = np.array([[1., 0, 0], [0, 1., 0]])
initial = initial.astype('float32')
initial = initial.flatten()
b_fc_loc2 = tf.Variable(initial_value=initial, name='b_fc_loc2')
# Define the connection of first layer
keep_prob = tf.placeholder(tf.float32)
h_fc_loc1 = tf.nn.tanh(tf.matmul(x, W_fc_loc1) + b_fc_loc1)
h_fc_loc1_drop = tf.nn.dropout(h_fc_loc1, keep_prob)
# Define the connection of second layer
h_fc_loc2 = tf.nn.tanh(tf.matmul(h_fc_loc1_drop, W_fc_loc2) + b_fc_loc2)
weights = []
biases = []
convs1 = []
dw_h_convs1 = OrderedDict()
convs2 = []
dw_h_convs2 = OrderedDict()
convs3 = []
dw_h_convs3 = OrderedDict()
convs_comb = []
dw_h_convs_comb = OrderedDict()
for layer in range(0, layers):
stddev = np.sqrt(2 / (filter_size**2 * features))
if layer == 0:
w1 = weight_variable(
[filter_size, filter_size, n_channels // 3, features], stddev)
else:
w1 = weight_variable(
[filter_size, filter_size, features, features], stddev)
b1 = bias_variable([features])
conv1 = conv2d(in_node1, w1, keep_prob)
dw_h_convs1[layer] = tf.nn.relu(conv1 + b1)
conv2 = conv2d(in_node2, w1, keep_prob)
dw_h_convs2[layer] = tf.nn.relu(conv2 + b1)
conv3 = conv2d(in_node3, w1, keep_prob)
dw_h_convs3[layer] = tf.nn.relu(conv3 + b1)
position_train, 2400)
Y_train = dense_to_one_hot(y_train, n_classes=6)
Y_valid = dense_to_one_hot(y_valid, n_classes=6)
Y_test = dense_to_one_hot(y_test, n_classes=6)
X_train = np.resize(X_train, (2400, 30, 4096, 1))
X_valid = np.resize(X_valid, (600, 30, 4096, 1))
X_test = np.resize(X_test, (600, 30, 4096, 1))
x = tf.placeholder(tf.float32, [None, 30, 4096, 1])
y = tf.placeholder(tf.float32, [None, 6])
keep_prob = tf.placeholder(tf.float32)
x_flat = tf.reshape(x, [-1, 30 * 4096])
x_trans = tf.reshape(x, [-1, 30, 4096, 1])
W_fc_loc1 = weight_variable([30 * 4096, 20])
b_fc_loc1 = bias_variable([20])
initial = np.array([0.5, 0])
initial = initial.astype('float32')
initial = initial.flatten()
W_fc_loc2 = weight_variable([20, 2])
b_fc_loc2 = bias_variable([2])
b_fc_loc2 = tf.Variable(initial_value=initial, name='b_fc_loc2')
h_fc_loc1 = tf.nn.tanh(tf.matmul(x_flat, W_fc_loc1) + b_fc_loc1)
h_fc_loc1_drop = tf.nn.dropout(h_fc_loc1, keep_prob)
h_fc_loc2 = tf.nn.tanh(tf.matmul(h_fc_loc1_drop, W_fc_loc2) + b_fc_loc2)
out_size = (10, 4096)