def build(self, input, is_dropout=False): #is_dropout 是否dropout
# 卷积层1 cov 5*5 6
W_conv1 = weight_variable([5, 5, tf.shape(input)[-1], 6]) #cov 5*5 6
b_conv1 = bias_variable([6])
h_conv1 = tf.nn.relu(conv2d(input, W_conv1) + b_conv1)
h_pool1 = avg_pooling(h_conv1)
# 卷积层2 cov 5*5 6
W_conv2 = weight_variable([5, 5, 6, 6])
b_conv2 = bias_variable([6])
h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)
h_pool2 = avg_pooling(h_conv2)
#全连接1 120
W_fc1 = weight_variable([7 * 7 * 6, 120]) #卷积后图像大小
b_fc1 = bias_variable([120])
h_pool2_flat = tf.reshape(h_pool2, [-1, 7 * 7 * 6]) #需要将卷积后的拉伸为一列
h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1)
if is_dropout: h_fc1 = tf.nn.dropout(h_fc1, 0.5)
# 全连接2 84
W_fc2 = weight_variable([120, 84]) # 卷积后图像大小
b_fc2 = bias_variable([84])
h_fc2 = tf.nn.relu(tf.matmul(h_fc1, W_fc2) + b_fc2)
if is_dropout: h_fc2 = tf.nn.dropout(h_fc2, 0.5)
# output 10
W_fc3 = weight_variable([84, 10]) # 卷积后图像大小
b_fc3 = bias_variable([10])
h_fc3 = tf.nn.relu(tf.matmul(h_fc2, W_fc3) + b_fc3)
return h_fc3
def _build_discriminator(inputs, data_shape, nplanes):
diff_input = inputs['diff_img']
keep_prob = inputs['keep_prob']
g1_weights = tf.get_variable('g1_weights', [5, 5, 3, nplanes])
g1_bias = tf.get_variable('g1_bias', [nplanes])
g1 = tf.nn.relu(utils.conv2d(g1_weights, bias=g1_bias)(diff_input))
g2_weights = tf.get_variable('g2_weights', [5, 5, nplanes, nplanes])
g2_bias = tf.get_variable('g2_bias', [nplanes])
g2 = tf.nn.relu(utils.conv2d(g2_weights, bias=g2_bias)(g1))
flattened = tf.reshape(g2, [-1, data_shape[0] * data_shape[1] * nplanes])
dropout = tf.nn.dropout(flattened, keep_prob)
prob_weights = tf.get_variable('prob_weights', [data_shape[0] * data_shape[1] * nplanes, 1])
prob_biases = tf.get_variable('prob_bias', [1])
y_logits = utils.dense(prob_weights, bias=prob_biases)(dropout)
y = tf.nn.sigmoid(y_logits, name='prob_real')
#class_weights = tf.get_variable('class_weights', [data_shape[0] * data_shape[1] * nplanes, 10])
#class_biases = tf.get_variable('class_bias', [1])
#class_logits = utils.dense(class_weights, bias=class_biases)(dropout)
#class_prob = tf.nn.softmax(class_logits, name='logits_class')
return {'prob': y, 'prob_logits': y_logits}#, 'prob_class': class_prob, 'logits_class': class_logits}
def discriminative(self, images, reuse=False):
with tf.variable_scope('discriminative') as scope:
if reuse:
scope.reuse_variables()
if self.run_flags.run == 'train':
is_training = True
else:
is_training = False
conv1 = lrelu(batch_norm(conv2d(images, output_dim=64, kernel=7, stride=1, name='d_conv1'), \
is_training=is_training, name='g_conv1_bn')) # 128 x 128 x 64
conv2 = lrelu(batch_norm(conv2d(conv1, output_dim=64, kernel=7, stride=2, name='d_conv2'), \
is_training=is_training, name='g_conv2_bn')) # 64 x 64 x 64
conv3 = lrelu(batch_norm(conv2d(conv2, output_dim=32, kernel=3, stride=2, name='d_conv3'), \
is_training=is_training, name='g_conv3_bn')) # 32 x 32 x 32
conv4 = lrelu(batch_norm(conv2d(conv3, output_dim=1, kernel=3, stride=2, name='d_conv4'), \
is_training=is_training, name='g_conv4_bn')) # 16 x 16 x 1
# conv1 = lrelu(conv2d(images, output_dim=64, kernel=7, stride=1, name='d_conv1')) # 128 x 128 x 64
#
# conv2 = lrelu(conv2d(conv1, output_dim=64, kernel=7, stride=2, name='d_conv2')) # 64 x 64 x 64
#
# conv3 = lrelu(conv2d(conv2, output_dim=32, kernel=3, stride=2, name='d_conv3')) # 32 x 32 x 32
#
# conv4 = lrelu(conv2d(conv3, output_dim=1, kernel=3, stride=2, name='d_conv4')) # 16 x 16 x 1
fc = tf.reshape(conv4, [-1, 16 * 16 * 1])
fc = ful_connect(fc, output_size=1, name='d_fc')
return fc
def build_cnn_model(self):
self.imgs = tf.placeholder('float32', [self.batch_size, self.input_dims])
self.img_reshape = tf.reshape(self.imgs, [self.batch_size, self.w, self.h, self.channel])
if self.synthetic:
self.layer_out['l1'], self.var['l1_w'], self.var['l1_b'], self.synthetic_grad['l1'] = conv2d(self.img_reshape, 128, [5,5], [1,1],
self.weight_initializer, self.bias_initializer, synthetic=True, batch_norm=True, activation_fn=tf.nn.relu, name='l1_con2d')
self.layer_out['l1_pool'] = pooling(self.layer_out['l1'], kernel_size=[3,3], stride=[1,1], type='max')
self.layer_out['l2'], self.var['l2_w'], self.var['l2_b'], self.synthetic_grad['l2'] = conv2d(self.layer_out['l1_pool'], 128, [5,5], [1,1],
self.weight_initializer, self.bias_initializer, synthetic=True, batch_norm=True, activation_fn=tf.nn.relu, name='l2_con2d')
self.layer_out['l2_pool'] = pooling(self.layer_out['l2'], kernel_size=[3,3], stride=[1,1], type='average')
self.layer_out['l3'], self.var['l3_w'], self.var['l3_b'], self.synthetic_grad['l3'] = conv2d(self.layer_out['l2_pool'], 128, [5,5], [1,1],
self.weight_initializer, self.bias_initializer, synthetic=True, batch_norm=True, activation_fn=tf.nn.relu, name='l3_con2d')
self.layer_out['l3_pool'] = pooling(self.layer_out['l3'], kernel_size=[3,3], stride=[1,1], type='average')
self.layer_out['l3_reshape'] = tf.reshape(self.layer_out['l3_pool'], [self.batch_size, -1])
self.layer_out['l4'], self.var['l4_w'], self.var['l4_b'], self.synthetic_grad['l4'] = linear(self.layer_out['l3_reshape'], self.output_size,
self.weight_initializer, self.bias_initializer, synthetic=True, activation_fn=tf.nn.relu, name='l4_linear')
else:
self.layer_out['l1'], self.var['l1_w'], self.var['l1_b'] = conv2d(self.img_reshape, 128, [5,5], [1,1],
self.weight_initializer, self.bias_initializer, batch_norm=True, activation_fn=tf.nn.relu, name='l1_con2d')
self.layer_out['l1_pool'] = pooling(self.layer_out['l1'], kernel_size=[3,3], stride=[1,1], type='max')
self.layer_out['l2'], self.var['l2_w'], self.var['l2_b'] = conv2d(self.layer_out['l1_pool'], 128, [5,5], [1,1],
self.weight_initializer, self.bias_initializer, batch_norm=True, activation_fn=tf.nn.relu, name='l2_con2d')
self.layer_out['l2_pool'] = pooling(self.layer_out['l2'], kernel_size=[3,3], stride=[1,1], type='average')
self.layer_out['l3'], self.var['l3_w'], self.var['l3_b'] = conv2d(self.layer_out['l2_pool'], 128, [5,5], [1,1],
self.weight_initializer, self.bias_initializer, batch_norm=True, activation_fn=tf.nn.relu, name='l3_con2d')
self.layer_out['l3_pool'] = pooling(self.layer_out['l3'], kernel_size=[3,3], stride=[1,1], type='average')
self.layer_out['l3_reshape'] = tf.reshape(self.layer_out['l3_pool'], [self.batch_size, -1])
self.layer_out['l4'], self.var['l4_w'], self.var['l4_b'] = linear(self.layer_out['l3_reshape'], self.output_size,
self.weight_initializer, self.bias_initializer, activation_fn=tf.nn.relu, name='l4_linear')
self.out_logit = tf.nn.softmax(self.layer_out['l4'])
self.out_argmax = tf.argmax(self.out_logit, 1)
self.labels = tf.placeholder('int32', [self.batch_size])
self.loss_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(self.layer_out['l4'], self.labels)
self.loss = tf.reduce_sum(self.loss_entropy)/self.batch_size
if self.synthetic:
self.grad_output['l1'] = tf.gradients(self.loss, self.layer_out['l1'])
self.grad_output['l2'] = tf.gradients(self.loss, self.layer_out['l2'])
self.grad_output['l3'] = tf.gradients(self.loss, self.layer_out['l3'])
self.grad_output['l4'] = tf.gradients(self.loss, self.layer_out['l4'])
for k in self.grad_output.keys():
self.grad_loss.append(tf.reduce_sum(tf.square(self.synthetic_grad[k]-self.grad_output[k])))
self.grad_total_loss = sum(self.grad_loss)
def generator(self, cond):
with tf.variable_scope("gen"):
feature = conf.conv_channel_base
e1 = conv2d(cond, feature, name="e1")
e2 = batch_norm(conv2d(lrelu(e1), feature*2, name="e2"), "e2")
e3 = batch_norm(conv2d(lrelu(e2), feature*4, name="e3"), "e3")
e4 = batch_norm(conv2d(lrelu(e3), feature*8, name="e4"), "e4")
e5 = batch_norm(conv2d(lrelu(e4), feature*8, name="e5"), "e5")
e6 = batch_norm(conv2d(lrelu(e5), feature*8, name="e6"), "e6")
e7 = batch_norm(conv2d(lrelu(e6), feature*8, name="e7"), "e7")
e8 = batch_norm(conv2d(lrelu(e7), feature*8, name="e8"), "e8")
d1 = deconv2d(tf.nn.relu(e8), [1,2,2,feature*8], name="d1")
d1 = tf.concat(3, [tf.nn.dropout(batch_norm(d1, "d1"), 0.5), e7])
d2 = deconv2d(tf.nn.relu(d1), [1,4,4,feature*8], name="d2")
d2 = tf.concat(3, [tf.nn.dropout(batch_norm(d2, "d2"), 0.5), e6])
d3 = deconv2d(tf.nn.relu(d2), [1,8,8,feature*8], name="d3")
d3 = tf.concat(3, [tf.nn.dropout(batch_norm(d3, "d3"), 0.5), e5])
d4 = deconv2d(tf.nn.relu(d3), [1,16,16,feature*8], name="d4")
d4 = tf.concat(3, [batch_norm(d4, "d4"), e4])
d5 = deconv2d(tf.nn.relu(d4), [1,32,32,feature*4], name="d5")
d5 = tf.concat(3, [batch_norm(d5, "d5"), e3])
d6 = deconv2d(tf.nn.relu(d5), [1,64,64,feature*2], name="d6")
d6 = tf.concat(3, [batch_norm(d6, "d6"), e2])
d7 = deconv2d(tf.nn.relu(d6), [1,128,128,feature], name="d7")
d7 = tf.concat(3, [batch_norm(d7, "d7"), e1])
d8 = deconv2d(tf.nn.relu(d7), [1,256,256,conf.img_channel], name="d8")
return tf.nn.tanh(d8)
def inference(self,images,is_training,weight_decay=1e-5):
self.image_size = images.get_shape()[1:3]
conv1 = atrous_conv2d(images,48,[3,3],'conv1',rate=2,weight_decay=weight_decay,use_xavier =True,
stddev=1e-3,is_training= is_training,bn=True,activation_fn=tf.nn.relu)
pool1 = max_pool2d(conv1,[3,3],'pool1',[1,1],padding='SAME')
conv2 = atrous_conv2d(pool1,48,[3,3],'conv2',rate=2,weight_decay=weight_decay,use_xavier=True,
stddev=1e-3,is_training=is_training,bn=True,activation_fn=tf.nn.relu)
pool2 = max_pool2d(conv2,[3,3],'pool12',[1,1],padding='SAME')
conv3 = atrous_conv2d(pool2,96, [3,3], 'conv3', rate=2, weight_decay=weight_decay, use_xavier=True,
stddev=1e-3, is_training=is_training, bn=True, activation_fn=tf.nn.relu)
pool3 = max_pool2d(conv3, [3,3], 'pool13', [1, 1], padding='SAME')
conv4 = atrous_conv2d(pool3,128,[3,3],'conv4',rate=2,weight_decay=weight_decay,use_xavier=True,
stddev=1e-3,is_training=is_training,bn=True,activation_fn= tf.nn.relu)
pool4 = max_pool2d(conv4, [3,3], 'pool14', [1, 1], padding='SAME')
conv5 = conv2d(pool4,N_CLASSES,[3,3],'conv5',[1,1],weight_decay=weight_decay,use_xavier=True,
stddev=1e-3,is_training=is_training,bn=False,activation_fn= None)
logits = conv5
return logits
def back_propagate(self):
"""
Refine parameters of this layer with residuals from next layer
"""
# compute gradient
partial_theta = numpy.asarray(map(
lambda i: numpy.rot90(conv2d(
reduce(lambda res, x: res + self.x_imgs[x], self.connections[i], 0),
numpy.rot90(self.delta[i], 2)
), 2),
xrange(0, len(self.connections))
))
parital_b = numpy.asarray(map(lambda x: numpy.sum(x), self.delta))
# if previous layer is input layer, then do nothing
if isinstance(self.prev_layer, InputLayer):
return
# compute residuals of previous pooling layer
if not self.prev_layer.connections:
self.prev_layer.connections = [[] for i in xrange(0, len(self.x_imgs))]
for i in xrange(0, len(self.connections)):
for c in self.connections[i]:
self.prev_layer.connections[c].append(i)
conv_full_res = numpy.asarray(map(
lambda i: conv2d(
self.delta[i],
numpy.rot90(self.theta[i], 2),
border_mode='full'
),
xrange(0, len(self.theta))
))
self.prev_layer.delta = numpy.asarray(map(
lambda i: dtanh(self.x_imgs[i])*reduce(
lambda res, x: res + conv_full_res[x],
self.prev_layer.connections[i],
0
),
xrange(0, len(self.x_imgs))
))
# update weights and bias
self.theta -= self.learning_rate*partial_theta
self.b -= self.learning_rate*parital_b
# continue back propagating
self.prev_layer.back_propagate()
def _build_autoencoder(inputs, data_shape, num_planes, latent_dim):
base_img = inputs['base_img']
class_cond = inputs['class_cond']
# We don't actually need a bias here as we are learning a bitplane per class anyways
class_weights = tf.get_variable('class_weights', [10, data_shape[0] * data_shape[1]])
class_vec = utils.dense(class_weights)(class_cond)
class_plane = tf.nn.relu(tf.reshape(class_vec, [-1, data_shape[0], data_shape[1], 1]))
# Now concatenate the tensors
stacked_input = tf.concat([base_img, class_plane], axis=3)
c1_weights = tf.get_variable('d1_weights', [7, 7, data_shape[2] + 1, num_planes])
c1_bias = tf.get_variable('d1_bias', [num_planes])
c1 = utils.conv2d(c1_weights, bias=c1_bias)(stacked_input)
c2_weights = tf.get_variable('d2_weights', [7, 7, num_planes, num_planes])
c2_bias = tf.get_variable('d2_bias', [num_planes])
c2 = utils.conv2d(c2_weights, bias=c2_bias)(c1)
c2_dropout = tf.nn.dropout(c2, inputs['keep_prob'])
c3_weights = tf.get_variable('d3_weights', [7, 7, num_planes, num_planes])
c3_bias = tf.get_variable('d3_bias', [num_planes])
c3 = utils.conv2d(c3_weights, bias=c3_bias)(c2_dropout)
c4_weights = tf.get_variable('d4_weights', [7, 7, num_planes, num_planes])
c4_bias = tf.get_variable('d4_bias', [num_planes])
c4 = utils.conv2d(c4_weights, bias=c4_bias)(c3)
c4_dropout = tf.nn.dropout(c4, inputs['keep_prob'])
c5_weights = tf.get_variable('d5_weights', [7, 7, num_planes, 3])
c5_bias = tf.get_variable('d5_bias', [3])
c5 = utils.conv2d(c5_weights, bias=c5_bias)(c4_dropout)
diff_img = c5
return {'diff_img': diff_img}
def output(self):
"""
Generate output of this layer
"""
self.x_imgs = self.prev_layer.output()
return numpy.asarray(map(
lambda i: tanh(self.b[i] + reduce(
lambda res, j: res + conv2d(self.x_imgs[j], self.theta[i]),
self.connections[i],
0
)),
xrange(0, len(self.connections))
))
def inference(examples):
# CONV1
with tf.variable_scope('conv1') as scope:
# conv weights [filter_height, filter_width, filter_depth, num_filters]
kernel = _variable('weights', [CONV1_HEIGHT, CONV1_WIDTH, 1, CONV1_FILTERS], tf.contrib.layers.xavier_initializer_conv2d())
biases = _variable('biases', [CONV1_FILTERS], tf.constant_initializer(0.1))
conv = conv2d(examples, kernel)
conv1 = tf.nn.relu(conv + biases, name=scope.name)
_activation_summary(conv1)
# pool1 dim: [n, time, freq after pooling, num_filters]
pool1 = tf.nn.max_pool(conv1, ksize=[1, POOL1_HEIGHT, POOL1_WIDTH, 1],
strides=[1, POOL1_STRIDE_HEIGHT, POOL1_STRIDE_WIDTH, 1], padding='SAME', name='pool1')
## TODO: add batch norm 1 here
batch_norm1_object = BatchNorm(name='batch_norm1', shape=[CONV1_FILTERS])
batch_norm1 = batch_norm1_object(pool1)
# CONV2
with tf.variable_scope('conv2') as scope:
kernel = _variable('weights', [CONV2_HEIGHT, CONV2_WIDTH, CONV1_FILTERS, CONV2_FILTERS], tf.contrib.layers.xavier_initializer_conv2d())
biases = _variable('biases', [CONV2_FILTERS], tf.constant_initializer(0.1))
conv = conv2d(batch_norm1, kernel)
conv2 = tf.nn.relu(conv + biases, name=scope.name)
_activation_summary(conv2)
# POOL2
pool2 = tf.nn.max_pool(conv2, ksize=[1, POOL2_HEIGHT, POOL2_WIDTH, 1],
strides=[1, POOL2_STRIDE_HEIGHT, POOL2_STRIDE_WIDTH, 1], padding='SAME', name='pool2')
## TODO: add batch norm 2 here
batch_norm2_object = BatchNorm(name='batch_norm2', shape=[CONV2_FILTERS])
batch_norm2 = batch_norm2_object(pool2)
# FC3
with tf.variable_scope('fc3') as scope:
reshape = tf.reshape(batch_norm2, [BATCH_SIZE, -1])
dim = (DIM_TIME/POOL1_HEIGHT/POOL2_HEIGHT) * (DIM_FREQ/POOL1_WIDTH/POOL2_WIDTH) * CONV2_FILTERS
weights = _variable('weights', [dim, FC3_SIZE], tf.contrib.layers.xavier_initializer(), wd=config.fc_wd)
biases = _variable('biases', [FC3_SIZE], tf.constant_initializer(0.1))
fc3 = tf.nn.relu(tf.matmul(reshape, weights) + biases, name=scope.name)
_activation_summary(fc3)
# FC4
with tf.variable_scope('fc4') as scope:
weights = _variable('weights', [FC3_SIZE, FC4_SIZE], tf.contrib.layers.xavier_initializer(), wd=config.fc_wd)
biases = _variable('biases', [FC4_SIZE], tf.constant_initializer(0.1))
fc4 = tf.nn.relu(tf.matmul(fc3, weights) + biases, name=scope.name)
_activation_summary(fc4)
# FC5
with tf.variable_scope('fc5') as scope:
weights = _variable('weights', [FC4_SIZE, FC5_SIZE], tf.contrib.layers.xavier_initializer(), wd=config.fc_wd)
biases = _variable('biases', [FC5_SIZE], tf.constant_initializer(0.1))
fc5 = tf.nn.relu(tf.matmul(fc4, weights) + biases, name=scope.name)
_activation_summary(fc5)
# FC6
with tf.variable_scope('fc6') as scope:
weights = _variable('weights', [FC5_SIZE, FC6_SIZE], tf.contrib.layers.xavier_initializer(), wd=config.fc_wd)
biases = _variable('biases', [FC6_SIZE], tf.constant_initializer(0.1))
fc6 = tf.nn.relu(tf.matmul(fc5, weights) + biases, name=scope.name)
_activation_summary(fc6)
# softmax
with tf.variable_scope('softmax_linear') as scope:
weights = _variable('weights', [FC6_SIZE, NUM_CLASSES], tf.contrib.layers.xavier_initializer())
biases = _variable('biases', [NUM_CLASSES], tf.constant_initializer(0.0))
# shape of y_conv is (N,3)
softmax_linear = tf.add(tf.matmul(fc6, weights), biases, name=scope.name)
_activation_summary(softmax_linear)
return softmax_linear
