def discriminator(x_image, reuse=False, type='real'):
with tf.variable_scope('discriminator') as scope:
if (reuse):
tf.get_variable_scope().reuse_variables()
#First Conv and Pool Layers
W_conv1 = tf.get_variable('d_wconv1', [5, 5, 1, 8], initializer=tf.truncated_normal_initializer(stddev=0.02))
b_conv1 = tf.get_variable('d_bconv1', [8], initializer=tf.constant_initializer(0))
h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1)
h_pool1 = avg_pool_2x2(h_conv1)
#Second Conv and Pool Layers
W_conv2 = tf.get_variable('d_wconv2', [5, 5, 8, 16], initializer=tf.truncated_normal_initializer(stddev=0.02))
b_conv2 = tf.get_variable('d_bconv2', [16], initializer=tf.constant_initializer(0))
h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)
h_pool2 = avg_pool_2x2(h_conv2)
#First Fully Connected Layer
W_fc1 = tf.get_variable('d_wfc1', [7 * 7 * 16, 32], initializer=tf.truncated_normal_initializer(stddev=0.02))
b_fc1 = tf.get_variable('d_bfc1', [32], initializer=tf.constant_initializer(0))
h_pool2_flat = tf.reshape(h_pool2, [-1, 7*7*16])
h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1)
#Second Fully Connected Layer
W_fc2 = tf.get_variable('d_wfc2', [32, 1], initializer=tf.truncated_normal_initializer(stddev=0.02))
b_fc2 = tf.get_variable('d_bfc2', [1], initializer=tf.constant_initializer(0))
#Final Layer
y_conv=(tf.matmul(h_fc1, W_fc2) + b_fc2)
if type == 'real':
TensorMonitor.AddUserList(
real_d_x_image=x_image,
real_d_h_conv1=h_conv1,
real_d_h_pool1=h_pool1,
real_d_h_conv2=h_conv2,
real_d_h_pool2=h_pool2,
real_d_h_fc1=h_fc1,
real_d_y_conv=y_conv)
else:
TensorMonitor.AddUserList(
fake_d_x_image=x_image,
fake_d_h_conv1=h_conv1,
fake_d_h_pool1=h_pool1,
fake_d_h_conv2=h_conv2,
fake_d_h_pool2=h_pool2,
fake_d_h_fc1=h_fc1,
fake_d_y_conv=y_conv)
return y_conv
def fc_layers(self):
# fc1
with tf.name_scope('fc1') as scope:
shape = int(np.prod(self.pool5.get_shape()[1:]))
fc1w = tf.Variable(tf.truncated_normal([shape, 4096],
dtype=tf.float32,
stddev=1e-1),
name='weights')
fc1b = tf.Variable(tf.constant(1.0, shape=[4096],
dtype=tf.float32),
trainable=True,
name='biases')
pool5_flat = tf.reshape(self.pool5, [-1, shape])
fc1l = tf.nn.bias_add(tf.matmul(pool5_flat, fc1w), fc1b)
self.fc1 = tf.nn.relu(fc1l)
self.parameters += [fc1w, fc1b]
TensorMonitor.AddUserList(fc1=self.fc1)
# fc2
with tf.name_scope('fc2') as scope:
fc2w = tf.Variable(tf.truncated_normal([4096, 4096],
dtype=tf.float32,
stddev=1e-1),
name='weights')
fc2b = tf.Variable(tf.constant(1.0, shape=[4096],
dtype=tf.float32),
trainable=True,
name='biases')
fc2l = tf.nn.bias_add(tf.matmul(self.fc1, fc2w), fc2b)
self.fc2 = tf.nn.relu(fc2l)
self.parameters += [fc2w, fc2b]
TensorMonitor.AddUserList(fc2=self.fc2)
# fc3
with tf.name_scope('fc3') as scope:
fc3w = tf.Variable(tf.truncated_normal([4096, 1000],
dtype=tf.float32,
stddev=1e-1),
name='weights')
fc3b = tf.Variable(tf.constant(1.0, shape=[1000],
dtype=tf.float32),
trainable=True,
name='biases')
self.fc3l = tf.nn.bias_add(tf.matmul(self.fc2, fc3w), fc3b)
self.parameters += [fc3w, fc3b]
sess.run(init)
# Training
for i in range(1, num_steps + 1):
# Prepare Data
# Get the next batch of MNIST data (only images are needed, not labels)
batch_x, _ = mnist.train.next_batch(batch_size)
# Run optimization op (backprop) and cost op (to get loss value)
_, l = sess.run([optimizer, loss], feed_dict={X: batch_x})
# Display logs per step
if i % display_step == 0 or i == 1:
print('Step %i: Minibatch Loss: %f' % (i, l))
bb = batch_x[0]
cmd = TensorMonitor.Beat(sess, input={X: bb[np.newaxis, :]})
"""
for t in tf.get_default_graph().get_operations():
if (t.name == 'encoder/layer_2'):
print('-------------------', t.values())
bb = batch_x[0]
ttt = sess.run(t.values(), feed_dict={X:bb[np.newaxis,:]})
print(ttt)
break
"""
# Testing
# Encode and decode images from test set and visualize their reconstruction.
n = 4
canvas_orig = np.empty((28 * n, 28 * n))
canvas_recon = np.empty((28 * n, 28 * n))
for i in range(n):
def convlayers(self):
self.parameters = []
# zero-mean input
with tf.name_scope('preprocess') as scope:
mean = tf.constant([123.68, 116.779, 103.939],
dtype=tf.float32,
shape=[1, 1, 1, 3],
name='img_mean')
images = self.imgs - mean
# conv1_1
with tf.name_scope('conv1_1') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 3, 64],
dtype=tf.float32,
stddev=1e-1),
name='weights')
conv = tf.nn.conv2d(images, kernel, [1, 1, 1, 1], padding='SAME')
biases = tf.Variable(tf.constant(0.0, shape=[64],
dtype=tf.float32),
trainable=True,
name='biases')
out = tf.nn.bias_add(conv, biases)
self.conv1_1 = tf.nn.relu(out, name=scope)
self.parameters += [kernel, biases]
TensorMonitor.AddUserList(conv1_1=self.conv1_1)
# conv1_2
with tf.name_scope('conv1_2') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 64, 64],
dtype=tf.float32,
stddev=1e-1),
name='weights')
conv = tf.nn.conv2d(self.conv1_1,
kernel, [1, 1, 1, 1],
padding='SAME')
biases = tf.Variable(tf.constant(0.0, shape=[64],
dtype=tf.float32),
trainable=True,
name='biases')
out = tf.nn.bias_add(conv, biases)
self.conv1_2 = tf.nn.relu(out, name=scope)
self.parameters += [kernel, biases]
TensorMonitor.AddUserList(conv1_2=self.conv1_2)
# pool1
self.pool1 = tf.nn.max_pool(self.conv1_2,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME',
name='pool1')
# conv2_1
with tf.name_scope('conv2_1') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 64, 128],
dtype=tf.float32,
stddev=1e-1),
name='weights')
conv = tf.nn.conv2d(self.pool1,
kernel, [1, 1, 1, 1],
padding='SAME')
biases = tf.Variable(tf.constant(0.0,
shape=[128],
dtype=tf.float32),
trainable=True,
name='biases')
out = tf.nn.bias_add(conv, biases)
self.conv2_1 = tf.nn.relu(out, name=scope)
self.parameters += [kernel, biases]
TensorMonitor.AddUserList(conv2_1=self.conv2_1)
# conv2_2
with tf.name_scope('conv2_2') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 128, 128],
dtype=tf.float32,
stddev=1e-1),
name='weights')
conv = tf.nn.conv2d(self.conv2_1,
kernel, [1, 1, 1, 1],
padding='SAME')
biases = tf.Variable(tf.constant(0.0,
shape=[128],
dtype=tf.float32),
trainable=True,
name='biases')
out = tf.nn.bias_add(conv, biases)
self.conv2_2 = tf.nn.relu(out, name=scope)
self.parameters += [kernel, biases]
TensorMonitor.AddUserList(conv2_2=self.conv2_2)
# pool2
self.pool2 = tf.nn.max_pool(self.conv2_2,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME',
name='pool2')
# conv3_1
with tf.name_scope('conv3_1') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 128, 256],
dtype=tf.float32,
stddev=1e-1),
name='weights')
conv = tf.nn.conv2d(self.pool2,
kernel, [1, 1, 1, 1],
padding='SAME')
biases = tf.Variable(tf.constant(0.0,
shape=[256],
dtype=tf.float32),
trainable=True,
name='biases')
out = tf.nn.bias_add(conv, biases)
self.conv3_1 = tf.nn.relu(out, name=scope)
self.parameters += [kernel, biases]
TensorMonitor.AddUserList(conv3_1=self.conv3_1)
# conv3_2
with tf.name_scope('conv3_2') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 256, 256],
dtype=tf.float32,
stddev=1e-1),
name='weights')
conv = tf.nn.conv2d(self.conv3_1,
kernel, [1, 1, 1, 1],
padding='SAME')
biases = tf.Variable(tf.constant(0.0,
shape=[256],
dtype=tf.float32),
trainable=True,
name='biases')
out = tf.nn.bias_add(conv, biases)
self.conv3_2 = tf.nn.relu(out, name=scope)
self.parameters += [kernel, biases]
TensorMonitor.AddUserList(conv3_2=self.conv3_2)
# conv3_3
with tf.name_scope('conv3_3') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 256, 256],
dtype=tf.float32,
stddev=1e-1),
name='weights')
conv = tf.nn.conv2d(self.conv3_2,
kernel, [1, 1, 1, 1],
padding='SAME')
biases = tf.Variable(tf.constant(0.0,
shape=[256],
dtype=tf.float32),
trainable=True,
name='biases')
out = tf.nn.bias_add(conv, biases)
self.conv3_3 = tf.nn.relu(out, name=scope)
self.parameters += [kernel, biases]
TensorMonitor.AddUserList(conv3_3=self.conv3_3)
# pool3
self.pool3 = tf.nn.max_pool(self.conv3_3,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME',
name='pool3')
# conv4_1
with tf.name_scope('conv4_1') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 256, 512],
dtype=tf.float32,
stddev=1e-1),
name='weights')
conv = tf.nn.conv2d(self.pool3,
kernel, [1, 1, 1, 1],
padding='SAME')
biases = tf.Variable(tf.constant(0.0,
shape=[512],
dtype=tf.float32),
trainable=True,
name='biases')
out = tf.nn.bias_add(conv, biases)
self.conv4_1 = tf.nn.relu(out, name=scope)
self.parameters += [kernel, biases]
TensorMonitor.AddUserList(conv4_1=self.conv4_1)
# conv4_2
with tf.name_scope('conv4_2') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 512, 512],
dtype=tf.float32,
stddev=1e-1),
name='weights')
conv = tf.nn.conv2d(self.conv4_1,
kernel, [1, 1, 1, 1],
padding='SAME')
biases = tf.Variable(tf.constant(0.0,
shape=[512],
dtype=tf.float32),
trainable=True,
name='biases')
out = tf.nn.bias_add(conv, biases)
self.conv4_2 = tf.nn.relu(out, name=scope)
self.parameters += [kernel, biases]
TensorMonitor.AddUserList(conv4_2=self.conv4_2)
# conv4_3
with tf.name_scope('conv4_3') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 512, 512],
dtype=tf.float32,
stddev=1e-1),
name='weights')
conv = tf.nn.conv2d(self.conv4_2,
kernel, [1, 1, 1, 1],
padding='SAME')
biases = tf.Variable(tf.constant(0.0,
shape=[512],
dtype=tf.float32),
trainable=True,
name='biases')
out = tf.nn.bias_add(conv, biases)
self.conv4_3 = tf.nn.relu(out, name=scope)
self.parameters += [kernel, biases]
TensorMonitor.AddUserList(conv4_3=self.conv4_3)
# pool4
self.pool4 = tf.nn.max_pool(self.conv4_3,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME',
name='pool4')
# conv5_1
with tf.name_scope('conv5_1') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 512, 512],
dtype=tf.float32,
stddev=1e-1),
name='weights')
conv = tf.nn.conv2d(self.pool4,
kernel, [1, 1, 1, 1],
padding='SAME')
biases = tf.Variable(tf.constant(0.0,
shape=[512],
dtype=tf.float32),
trainable=True,
name='biases')
out = tf.nn.bias_add(conv, biases)
self.conv5_1 = tf.nn.relu(out, name=scope)
self.parameters += [kernel, biases]
TensorMonitor.AddUserList(conv5_1=self.conv5_1)
# conv5_2
with tf.name_scope('conv5_2') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 512, 512],
dtype=tf.float32,
stddev=1e-1),
name='weights')
conv = tf.nn.conv2d(self.conv5_1,
kernel, [1, 1, 1, 1],
padding='SAME')
biases = tf.Variable(tf.constant(0.0,
shape=[512],
dtype=tf.float32),
trainable=True,
name='biases')
out = tf.nn.bias_add(conv, biases)
self.conv5_2 = tf.nn.relu(out, name=scope)
self.parameters += [kernel, biases]
TensorMonitor.AddUserList(conv5_2=self.conv5_2)
# conv5_3
with tf.name_scope('conv5_3') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 512, 512],
dtype=tf.float32,
stddev=1e-1),
name='weights')
conv = tf.nn.conv2d(self.conv5_2,
kernel, [1, 1, 1, 1],
padding='SAME')
biases = tf.Variable(tf.constant(0.0,
shape=[512],
dtype=tf.float32),
trainable=True,
name='biases')
out = tf.nn.bias_add(conv, biases)
self.conv5_3 = tf.nn.relu(out, name=scope)
self.parameters += [kernel, biases]
TensorMonitor.AddUserList(conv5_3=self.conv5_3)
# pool5
self.pool5 = tf.nn.max_pool(self.conv5_3,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME',
name='pool4')
fc3b = tf.Variable(tf.constant(1.0, shape=[1000],
dtype=tf.float32),
trainable=True,
name='biases')
self.fc3l = tf.nn.bias_add(tf.matmul(self.fc2, fc3w), fc3b)
self.parameters += [fc3w, fc3b]
def load_weights(self, weight_file, sess):
weights = np.load(weight_file)
keys = sorted(weights.keys())
for i, k in enumerate(keys):
print('????', i, k, np.shape(weights[k]))
sess.run(self.parameters[i].assign(weights[k]))
if __name__ == '__main__':
sess = tf.Session()
imgs = tf.placeholder(tf.float32, [None, 224, 224, 3])
vgg = vgg16(imgs, 'vgg16_weights.npz', sess)
img1 = imread('laska.png', mode='RGB')
img1 = imresize(img1, (224, 224))
for i in range(3000):
prob = sess.run(vgg.probs, feed_dict={vgg.imgs: [img1]})[0]
cmd = TensorMonitor.Beat(sess, input={vgg.imgs: [img1]})
preds = (np.argsort(prob)[::-1])[0:5]
for p in preds:
print("Final result: this is ", class_names[p], prob[p])
def generator(z, batch_size, z_dim, reuse=False):
with tf.variable_scope('generator') as scope:
if (reuse):
tf.get_variable_scope().reuse_variables()
g_dim = 64 #Number of filters of first layer of generator
c_dim = 1 #Color dimension of output (MNIST is grayscale, so c_dim = 1 for us)
s = 28 #Output size of the image
s2, s4, s8, s16 = int(s/2), int(s/4), int(s/8), int(s/16) #We want to slowly upscale the image, so these values will help
#make that change gradual.
h0 = tf.reshape(z, [batch_size, s16+1, s16+1, 25])
h0 = tf.nn.relu(h0)
#Dimensions of h0 = batch_size x 2 x 2 x 25
#First DeConv Layer
output1_shape = [batch_size, s8, s8, g_dim*4]
W_conv1 = tf.get_variable('g_wconv1', [5, 5, output1_shape[-1], int(h0.get_shape()[-1])],
initializer=tf.truncated_normal_initializer(stddev=0.1))
b_conv1 = tf.get_variable('g_bconv1', [output1_shape[-1]], initializer=tf.constant_initializer(.1))
H_conv1 = tf.nn.conv2d_transpose(h0, W_conv1, output_shape=output1_shape, strides=[1, 2, 2, 1], padding='SAME')
H_conv1 = tf.contrib.layers.batch_norm(inputs = H_conv1, center=True, scale=True, is_training=True, scope="g_bn1")
H_conv1 = tf.nn.relu(H_conv1)
#Dimensions of H_conv1 = batch_size x 3 x 3 x 256
#Second DeConv Layer
output2_shape = [batch_size, s4 - 1, s4 - 1, g_dim*2]
W_conv2 = tf.get_variable('g_wconv2', [5, 5, output2_shape[-1], int(H_conv1.get_shape()[-1])],
initializer=tf.truncated_normal_initializer(stddev=0.1))
b_conv2 = tf.get_variable('g_bconv2', [output2_shape[-1]], initializer=tf.constant_initializer(.1))
H_conv2 = tf.nn.conv2d_transpose(H_conv1, W_conv2, output_shape=output2_shape, strides=[1, 2, 2, 1], padding='SAME')
H_conv2 = tf.contrib.layers.batch_norm(inputs = H_conv2, center=True, scale=True, is_training=True, scope="g_bn2")
H_conv2 = tf.nn.relu(H_conv2)
#Dimensions of H_conv2 = batch_size x 6 x 6 x 128
#Third DeConv Layer
output3_shape = [batch_size, s2 - 2, s2 - 2, g_dim*1]
W_conv3 = tf.get_variable('g_wconv3', [5, 5, output3_shape[-1], int(H_conv2.get_shape()[-1])],
initializer=tf.truncated_normal_initializer(stddev=0.1))
b_conv3 = tf.get_variable('g_bconv3', [output3_shape[-1]], initializer=tf.constant_initializer(.1))
H_conv3 = tf.nn.conv2d_transpose(H_conv2, W_conv3, output_shape=output3_shape, strides=[1, 2, 2, 1], padding='SAME')
H_conv3 = tf.contrib.layers.batch_norm(inputs = H_conv3, center=True, scale=True, is_training=True, scope="g_bn3")
H_conv3 = tf.nn.relu(H_conv3)
#Dimensions of H_conv3 = batch_size x 12 x 12 x 64
#Fourth DeConv Layer
output4_shape = [batch_size, s, s, c_dim]
W_conv4 = tf.get_variable('g_wconv4', [5, 5, output4_shape[-1], int(H_conv3.get_shape()[-1])],
initializer=tf.truncated_normal_initializer(stddev=0.1))
b_conv4 = tf.get_variable('g_bconv4', [output4_shape[-1]], initializer=tf.constant_initializer(.1))
H_conv4 = tf.nn.conv2d_transpose(H_conv3, W_conv4, output_shape=output4_shape, strides=[1, 2, 2, 1], padding='VALID')
H_conv4 = tf.nn.tanh(H_conv4)
#Dimensions of H_conv4 = batch_size x 28 x 28 x 1
TensorMonitor.AddUserList(
g_z=z,
g_h0=h0,
g_h_conv1=H_conv1,
g_h_conv2=H_conv2,
g_h_conv3=H_conv3,
g_h_conv4=H_conv4)
return H_conv4
tf.reset_default_graph() #Since we changed our batch size (from 1 to 16), we need to reset our Tensorflow graph
sess = tf.Session()
x_placeholder = tf.placeholder("float", shape = [None,28,28,1]) #Placeholder for input images to the discriminator
z_placeholder = tf.placeholder(tf.float32, [None, z_dimensions]) #Placeholder for input noise vectors to the generator
# One of the trickiest parts about understanding GANs is that the loss function is a little bit more complex than that of a traditional CNN classifiers (For those, a simple MSE or Hinge Loss would do the trick). If you think back to the introduction, a GAN can be thought of as a zero sum minimax game. The generator is constantly improving to produce more and more realistic images, while the discriminator is trying to get better and better at distinguishing between real and generated images. This means that we need to formulate loss functions that affect both networks. Let’s take a look at the inputs and outputs of our networks.
# In[13]:
Dx = discriminator(x_placeholder, type='real') #Dx will hold discriminator outputs (unnormalized) for the real MNIST images
Gz = generator(z_placeholder, batch_size, z_dimensions) #Gz holds the generated images
Dg = discriminator(Gz, reuse=True, type='fake') #Dg will hold discriminator outputs (unnormalized) for generated images
TensorMonitor.AddUserList(Dx=Dx)
TensorMonitor.AddUserList(Dg=Dg)
# So, let’s first think about what we want out of our networks. We want the generator network to create images that will fool the discriminator. The generator wants the discriminator to output a 1 (positive example). Therefore, we want to compute the loss between the Dg and label of 1. This can be done through the tf.nn.sigmoid_cross_entropy_with_logits function. This means that the cross entropy loss will be taken between the two arguments. The "with_logits" component means that the function will operate on unscaled values. Basically, this means that instead of using a softmax function to squish the output activations to probability values from 0 to 1, we simply return the unscaled value of the matrix multiplication. Take a look at the last line of our discriminator. There's no softmax or sigmoid layer at the end.
# The reduce mean function just takes the mean value of all of the components in the matrixx returned by the cross entropy function. This is just a way of reducing the loss to a single scalar value, instead of a vector or matrix.
# In[14]:
g_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=Dg, labels=tf.ones_like(Dg)))
# Now, let’s think about the discriminator’s point of view. Its goal is to just get the correct labels (output 1 for each MNIST digit and 0 for the generated ones). We’d like to compute the loss between Dx and the correct label of 1 as well as the loss between Dg and the correct label of 0.
# In[15]:
d_loss_real = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=Dx, labels=tf.ones_like(Dx)))
from TensorMonitor.tensor_manager import TensorMonitor
N_A = 30
step_count = 0
mu = tf.placeholder(tf.float32, [1], 'mu')
sigma = tf.placeholder(tf.float32, [1], 'sigma')
s = tf.placeholder(tf.float32, [None, N_A], 'A')
normal_dist = tf.distributions.Normal(mu, sigma)
prob = normal_dist.prob(s)
log_prob = normal_dist.log_prob(s)
#sample = tf.squeeze(normal_dist.sample(10000), axis=0)
TensorMonitor.AddUserList(
mu=mu,
sigma=sigma,
prob_a = prob,
log_prob_a = log_prob
)
# Initialize the variables (i.e. assign their default value)
init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
while True:
a = np.linspace(-3.14,3.14,N_A)
cmd = TensorMonitor.Beat(sess, input1={s:a[np.newaxis,:], mu:np.array([0]), sigma:np.array([100])},input2={s:a[np.newaxis,:], mu:np.array([0]), sigma:np.array([100])})
step_count += 1
print('step %d'%step_count)
if cmd == 'quit':