def defineEncodingVars(self): if(self.vggFile): npWeights = loadWeights(self.vggFile) else: print "Must load from weights" assert(0) self.W_conv1_1 = weight_variable_fromnp(npWeights["conv1_1_w"], "w_conv1_1") self.B_conv1_1 = weight_variable_fromnp(npWeights["conv1_1_b"], "b_conv1_1") self.W_conv1_2 = weight_variable_fromnp(npWeights["conv1_2_w"], "w_conv1_2") self.B_conv1_2 = weight_variable_fromnp(npWeights["conv1_2_b"], "b_conv1_2") self.W_conv2_1 = weight_variable_fromnp(npWeights["conv2_1_w"], "w_conv2_1") self.B_conv2_1 = weight_variable_fromnp(npWeights["conv2_1_b"], "b_conv2_1") self.W_conv2_2 = weight_variable_fromnp(npWeights["conv2_2_w"], "w_conv2_2") self.B_conv2_2 = weight_variable_fromnp(npWeights["conv2_2_b"], "b_conv2_2") self.W_conv3_1 = weight_variable_fromnp(npWeights["conv3_1_w"], "w_conv3_1") self.B_conv3_1 = weight_variable_fromnp(npWeights["conv3_1_b"], "b_conv3_1") self.W_conv3_2 = weight_variable_fromnp(npWeights["conv3_2_w"], "w_conv3_2") self.B_conv3_2 = weight_variable_fromnp(npWeights["conv3_2_b"], "b_conv3_2") self.W_conv3_3 = weight_variable_fromnp(npWeights["conv3_3_w"], "w_conv3_3") self.B_conv3_3 = weight_variable_fromnp(npWeights["conv3_3_b"], "b_conv3_3") self.W_conv4_1 = weight_variable_fromnp(npWeights["conv4_1_w"], "w_conv4_1") self.B_conv4_1 = weight_variable_fromnp(npWeights["conv4_1_b"], "b_conv4_1") self.W_conv4_2 = weight_variable_fromnp(npWeights["conv4_2_w"], "w_conv4_2") self.B_conv4_2 = weight_variable_fromnp(npWeights["conv4_2_b"], "b_conv4_2") self.W_conv4_3 = weight_variable_fromnp(npWeights["conv4_3_w"], "w_conv4_3") self.B_conv4_3 = weight_variable_fromnp(npWeights["conv4_3_b"], "b_conv4_3") self.W_conv5_1 = weight_variable_fromnp(npWeights["conv5_1_w"], "w_conv5_1") self.B_conv5_1 = weight_variable_fromnp(npWeights["conv5_1_b"], "b_conv5_1") self.W_conv5_2 = weight_variable_fromnp(npWeights["conv5_2_w"], "w_conv5_2") self.B_conv5_2 = weight_variable_fromnp(npWeights["conv5_2_b"], "b_conv5_2") self.W_conv5_3 = weight_variable_fromnp(npWeights["conv5_3_w"], "w_conv5_3") self.B_conv5_3 = weight_variable_fromnp(npWeights["conv5_3_b"], "b_conv5_3") #These are class vars, but defined here to load from npWeights self.class_fc_1_weight = weight_variable_fromnp(npWeights['fc6_w'], "class_fc_1_weight") self.class_fc_1_bias = weight_variable_fromnp(npWeights['fc6_b'], "class_fc_1_bias") self.class_fc_2_weight = weight_variable_fromnp(npWeights['fc7_w'], "class_fc_2_weight") self.class_fc_2_bias = weight_variable_fromnp(npWeights['fc7_b'], "class_fc_2_bias")
def buildModel(self, inputShape): if(self.vggFile): npWeights = loadWeights(self.vggFile) else: print "Must load from weights" assert(0) #Running on GPU with tf.device(self.device): with tf.name_scope("inputOps"): #Get convolution variables as placeholders self.inputImage = node_variable([self.batchSize, inputShape[0], inputShape[1], inputShape[2]], "inputImage") self.gt = node_variable([self.batchSize, self.numClasses], "gt") #Model variables for convolutions with tf.name_scope("Conv1Ops"): self.W_conv1_1 = weight_variable_fromnp(npWeights["conv1_1_w"], "w_conv1_1") self.B_conv1_1 = weight_variable_fromnp(npWeights["conv1_1_b"], "b_conv1_1") self.W_conv1_2 = weight_variable_fromnp(npWeights["conv1_2_w"], "w_conv1_2") self.B_conv1_2 = weight_variable_fromnp(npWeights["conv1_2_b"], "b_conv1_2") self.h_conv1_1 = tf.nn.relu(conv2d(self.inputImage, self.W_conv1_1, "conv1_1", stride=[1, 1, 1, 1]) + self.B_conv1_1) self.h_conv1_2 = tf.nn.relu(conv2d(self.h_conv1_1, self.W_conv1_2, "conv1_1", stride=[1, 1, 1, 1]) + self.B_conv1_2) self.h_pool1 = maxpool_2x2(self.h_conv1_2, "pool1") with tf.name_scope("Conv2Ops"): self.W_conv2_1 = weight_variable_fromnp(npWeights["conv2_1_w"], "w_conv2_1") self.B_conv2_1 = weight_variable_fromnp(npWeights["conv2_1_b"], "b_conv2_1") self.W_conv2_2 = weight_variable_fromnp(npWeights["conv2_2_w"], "w_conv2_2") self.B_conv2_2 = weight_variable_fromnp(npWeights["conv2_2_b"], "b_conv2_2") self.h_conv2_1 = tf.nn.relu(conv2d(self.h_pool1, self.W_conv2_1, "conv2_1") + self.B_conv2_1) self.h_conv2_2 = tf.nn.relu(conv2d(self.h_conv2_1, self.W_conv2_2, "conv2_2") + self.B_conv2_2) self.h_pool2 = maxpool_2x2(self.h_conv2_2, "pool2") with tf.name_scope("Conv3Ops"): self.W_conv3_1 = weight_variable_fromnp(npWeights["conv3_1_w"], "w_conv3_1") self.B_conv3_1 = weight_variable_fromnp(npWeights["conv3_1_b"], "b_conv3_1") self.W_conv3_2 = weight_variable_fromnp(npWeights["conv3_2_w"], "w_conv3_2") self.B_conv3_2 = weight_variable_fromnp(npWeights["conv3_2_b"], "b_conv3_2") self.W_conv3_3 = weight_variable_fromnp(npWeights["conv3_3_w"], "w_conv3_3") self.B_conv3_3 = weight_variable_fromnp(npWeights["conv3_3_b"], "b_conv3_3") self.h_conv3_1 = tf.nn.relu(conv2d(self.h_pool2, self.W_conv3_1, "conv3_1") + self.B_conv3_1) self.h_conv3_2 = tf.nn.relu(conv2d(self.h_conv3_1, self.W_conv3_2, "conv3_2") + self.B_conv3_2) self.h_conv3_3 = tf.nn.relu(conv2d(self.h_conv3_2, self.W_conv3_3, "conv3_2") + self.B_conv3_3) self.h_pool3 = maxpool_2x2(self.h_conv3_3, "pool3") with tf.name_scope("Conv4Ops"): self.W_conv4_1 = weight_variable_fromnp(npWeights["conv4_1_w"], "w_conv4_1") self.B_conv4_1 = weight_variable_fromnp(npWeights["conv4_1_b"], "b_conv4_1") self.W_conv4_2 = weight_variable_fromnp(npWeights["conv4_2_w"], "w_conv4_2") self.B_conv4_2 = weight_variable_fromnp(npWeights["conv4_2_b"], "b_conv4_2") self.W_conv4_3 = weight_variable_fromnp(npWeights["conv4_3_w"], "w_conv4_3") self.B_conv4_3 = weight_variable_fromnp(npWeights["conv4_3_b"], "b_conv4_3") self.h_conv4_1 = tf.nn.relu(conv2d(self.h_pool3, self.W_conv4_1, "conv4_1") + self.B_conv4_1) self.h_conv4_2 = tf.nn.relu(conv2d(self.h_conv4_1, self.W_conv4_2, "conv4_2") + self.B_conv4_2) self.h_conv4_3 = tf.nn.relu(conv2d(self.h_conv4_2, self.W_conv4_3, "conv4_2") + self.B_conv4_3) self.h_pool4 = maxpool_2x2(self.h_conv4_3, "pool4") with tf.name_scope("Conv5Ops"): self.W_conv5_1 = weight_variable_fromnp(npWeights["conv5_1_w"], "w_conv5_1") self.B_conv5_1 = weight_variable_fromnp(npWeights["conv5_1_b"], "b_conv5_1") self.W_conv5_2 = weight_variable_fromnp(npWeights["conv5_2_w"], "w_conv5_2") self.B_conv5_2 = weight_variable_fromnp(npWeights["conv5_2_b"], "b_conv5_2") self.W_conv5_3 = weight_variable_fromnp(npWeights["conv5_3_w"], "w_conv5_3") self.B_conv5_3 = weight_variable_fromnp(npWeights["conv5_3_b"], "b_conv5_3") self.h_conv5_1 = tf.nn.relu(conv2d(self.h_pool4, self.W_conv5_1, "conv5_1") + self.B_conv5_1) self.h_conv5_2 = tf.nn.relu(conv2d(self.h_conv5_1, self.W_conv5_2, "conv5_2") + self.B_conv5_2) self.h_conv5_3 = tf.nn.relu(conv2d(self.h_conv5_2, self.W_conv5_3, "conv5_2") + self.B_conv5_3) #16 comes from 4 2x2 pooling self.h_conv5_shape = [self.batchSize, inputShape[0]/16, inputShape[1]/16, 512] assert(inputShape[0]/16 == 14) with tf.name_scope("GAP"): self.h_gap = tf.reduce_mean(self.h_conv5_3, reduction_indices=[1, 2]) self.W_gap = weight_variable_xavier([512, self.numClasses], "w_gap", conv=False) self.B_gap = bias_variable([self.numClasses], "b_gap") self.est = tf.nn.softmax(tf.matmul(self.h_gap, self.W_gap)+self.B_gap) with tf.name_scope("CAM"): self.h_reshape_gap = tf.reshape(self.h_conv5_3, [self.batchSize*self.h_conv5_shape[1]*self.h_conv5_shape[2], -1]) self.flat_cam = tf.matmul(self.h_reshape_gap, self.W_gap) self.reshape_cam = tf.reshape(self.flat_cam, [self.batchSize, self.h_conv5_shape[1], self.h_conv5_shape[2], -1]) #self.softmax_cam = pixelSoftmax(self.reshape_cam) self.cam = tf.transpose(self.reshape_cam, [0, 3, 1, 2]) with tf.name_scope("Loss"): #Define loss self.loss = tf.reduce_mean(-tf.reduce_sum(self.gt * tf.log(self.est+self.epsilon), reduction_indices=[1])) with tf.name_scope("Opt"): #Define optimizer self.optimizerAll = tf.train.AdamOptimizer(self.learningRate, beta1=self.beta1, beta2=self.beta2, epsilon=self.epsilon).minimize(self.loss) #self.optimizerAll = tf.train.MomentumOptimizer(self.learningRate, momentum=self.beta1).minimize(self.loss) self.optimizerPre = tf.train.AdamOptimizer(self.learningRate, beta1=self.beta1, beta2=self.beta2, epsilon=self.epsilon).minimize(self.loss, var_list=[ self.W_gap, self.B_gap, ] ) with tf.name_scope("Metric"): self.correct = tf.equal(tf.argmax(self.gt, 1), tf.argmax(self.est, 1)) self.accuracy = tf.reduce_mean(tf.cast(self.correct, tf.float32)) #Cannot be on GPU (self.eval_vals, self.eval_idx) = tf.nn.top_k(self.est, k=11) #Summaries tf.scalar_summary('loss', self.loss, name="lossSum") tf.scalar_summary('accuracy', self.accuracy, name="accSum") tf.histogram_summary('input', self.inputImage, name="image") tf.histogram_summary('gt', self.gt, name="gt") #Conv layer histograms tf.histogram_summary('conv1_1', self.h_conv1_1, name="conv1_1") tf.histogram_summary('conv1_2', self.h_conv1_2, name="conv1_2") tf.histogram_summary('conv2_1', self.h_conv2_1, name="conv2_1") tf.histogram_summary('conv2_2', self.h_conv2_2, name="conv2_2") tf.histogram_summary('conv3_1', self.h_conv3_1, name="conv3_1") tf.histogram_summary('conv3_2', self.h_conv3_2, name="conv3_2") tf.histogram_summary('conv3_3', self.h_conv3_3, name="conv3_3") tf.histogram_summary('conv4_1', self.h_conv4_1, name="conv4_1") tf.histogram_summary('conv4_2', self.h_conv4_2, name="conv4_2") tf.histogram_summary('conv4_3', self.h_conv4_3, name="conv4_3") tf.histogram_summary('conv5_1', self.h_conv5_1, name="conv5_1") tf.histogram_summary('conv5_2', self.h_conv5_2, name="conv5_2") tf.histogram_summary('conv5_3', self.h_conv5_3, name="conv5_3") tf.histogram_summary('gap', self.h_gap, name="gap") tf.histogram_summary('est', self.est, name="est") #Weight and bias hists tf.histogram_summary('w_conv1_1', self.W_conv1_1, name="w_conv1_1") tf.histogram_summary('b_conv1_1', self.B_conv1_1, name="b_conv1_1") tf.histogram_summary('w_conv1_2', self.W_conv1_2, name="w_conv1_2") tf.histogram_summary('b_conv1_2', self.B_conv1_2, name="b_conv1_2") tf.histogram_summary('w_conv2_1', self.W_conv2_1, name="w_conv2_1") tf.histogram_summary('b_conv2_1', self.B_conv2_1, name="b_conv2_1") tf.histogram_summary('w_conv2_2', self.W_conv2_2, name="w_conv2_2") tf.histogram_summary('b_conv2_2', self.B_conv2_2, name="b_conv2_2") tf.histogram_summary('w_conv3_1', self.W_conv3_1, name="w_conv3_1") tf.histogram_summary('b_conv3_1', self.B_conv3_1, name="b_conv3_1") tf.histogram_summary('w_conv3_2', self.W_conv3_2, name="w_conv3_2") tf.histogram_summary('b_conv3_2', self.B_conv3_2, name="b_conv3_2") tf.histogram_summary('w_conv3_3', self.W_conv3_3, name="w_conv3_3") tf.histogram_summary('b_conv3_3', self.B_conv3_3, name="b_conv3_3") tf.histogram_summary('w_conv4_1', self.W_conv4_1, name="w_conv4_1") tf.histogram_summary('b_conv4_1', self.B_conv4_1, name="b_conv4_1") tf.histogram_summary('w_conv4_2', self.W_conv4_2, name="w_conv4_2") tf.histogram_summary('b_conv4_2', self.B_conv4_2, name="b_conv4_2") tf.histogram_summary('w_conv4_3', self.W_conv4_3, name="w_conv4_3") tf.histogram_summary('b_conv4_3', self.B_conv4_3, name="b_conv4_3") tf.histogram_summary('w_conv5_1', self.W_conv5_1, name="w_conv5_1") tf.histogram_summary('b_conv5_1', self.B_conv5_1, name="b_conv5_1") tf.histogram_summary('w_conv5_2', self.W_conv5_2, name="w_conv5_2") tf.histogram_summary('b_conv5_2', self.B_conv5_2, name="b_conv5_2") tf.histogram_summary('w_conv5_3', self.W_conv5_3, name="w_conv5_3") tf.histogram_summary('b_conv5_3', self.B_conv5_3, name="b_conv5_3") tf.histogram_summary('w_gap', self.W_gap, name="w_gap") tf.histogram_summary('b_gap', self.B_gap, name="w_gap")
def buildModel(self, inputShape): if (self.vggFile): npWeights = loadWeights(self.vggFile) else: print "Must load from weights" assert (0) #Running on GPU with tf.device(self.device): with tf.name_scope("inputOps"): #Get convolution variables as placeholders self.inputImage = node_variable([ self.batchSize, inputShape[0], inputShape[1], inputShape[2] ], "inputImage") self.gt = node_variable([self.batchSize, self.numClasses], "gt") self.notGt = 1 - self.gt with tf.name_scope("Conv1Ops"): self.W_conv1_1 = weight_variable_fromnp( npWeights["conv1_1_w"], "w_conv1_1") self.B_conv1_1 = weight_variable_fromnp( npWeights["conv1_1_b"], "b_conv1_1") self.W_conv1_2 = weight_variable_fromnp( npWeights["conv1_2_w"], "w_conv1_2") self.B_conv1_2 = weight_variable_fromnp( npWeights["conv1_2_b"], "b_conv1_2") self.h_conv1_1 = tf.nn.relu( conv2d(self.inputImage, self.W_conv1_1, "conv1_1", stride=[1, 1, 1, 1]) + self.B_conv1_1) self.h_conv1_2 = tf.nn.relu( conv2d(self.h_conv1_1, self.W_conv1_2, "conv1_1", stride=[1, 1, 1, 1]) + self.B_conv1_2) self.h_pool1 = maxpool_2x2(self.h_conv1_2, "pool1") with tf.name_scope("Conv2Ops"): self.W_conv2_1 = weight_variable_fromnp( npWeights["conv2_1_w"], "w_conv2_1") self.B_conv2_1 = weight_variable_fromnp( npWeights["conv2_1_b"], "b_conv2_1") self.W_conv2_2 = weight_variable_fromnp( npWeights["conv2_2_w"], "w_conv2_2") self.B_conv2_2 = weight_variable_fromnp( npWeights["conv2_2_b"], "b_conv2_2") self.h_conv2_1 = tf.nn.relu( conv2d(self.h_pool1, self.W_conv2_1, "conv2_1") + self.B_conv2_1) self.h_conv2_2 = tf.nn.relu( conv2d(self.h_conv2_1, self.W_conv2_2, "conv2_2") + self.B_conv2_2) self.h_pool2 = maxpool_2x2(self.h_conv2_2, "pool2") with tf.name_scope("Conv3Ops"): self.W_conv3_1 = weight_variable_fromnp( npWeights["conv3_1_w"], "w_conv3_1") self.B_conv3_1 = weight_variable_fromnp( npWeights["conv3_1_b"], "b_conv3_1") self.W_conv3_2 = weight_variable_fromnp( npWeights["conv3_2_w"], "w_conv3_2") self.B_conv3_2 = weight_variable_fromnp( npWeights["conv3_2_b"], "b_conv3_2") self.W_conv3_3 = weight_variable_fromnp( npWeights["conv3_3_w"], "w_conv3_3") self.B_conv3_3 = weight_variable_fromnp( npWeights["conv3_3_b"], "b_conv3_3") self.h_conv3_1 = tf.nn.relu( conv2d(self.h_pool2, self.W_conv3_1, "conv3_1") + self.B_conv3_1) self.h_conv3_2 = tf.nn.relu( conv2d(self.h_conv3_1, self.W_conv3_2, "conv3_2") + self.B_conv3_2) self.h_conv3_3 = tf.nn.relu( conv2d(self.h_conv3_2, self.W_conv3_3, "conv3_2") + self.B_conv3_3) self.h_pool3 = maxpool_2x2(self.h_conv3_3, "pool3") with tf.name_scope("Conv4Ops"): self.W_conv4_1 = weight_variable_fromnp( npWeights["conv4_1_w"], "w_conv4_1") self.B_conv4_1 = weight_variable_fromnp( npWeights["conv4_1_b"], "b_conv4_1") self.W_conv4_2 = weight_variable_fromnp( npWeights["conv4_2_w"], "w_conv4_2") self.B_conv4_2 = weight_variable_fromnp( npWeights["conv4_2_b"], "b_conv4_2") self.W_conv4_3 = weight_variable_fromnp( npWeights["conv4_3_w"], "w_conv4_3") self.B_conv4_3 = weight_variable_fromnp( npWeights["conv4_3_b"], "b_conv4_3") self.h_conv4_1 = tf.nn.relu( conv2d(self.h_pool3, self.W_conv4_1, "conv4_1") + self.B_conv4_1) self.h_conv4_2 = tf.nn.relu( conv2d(self.h_conv4_1, self.W_conv4_2, "conv4_2") + self.B_conv4_2) self.h_conv4_3 = tf.nn.relu( conv2d(self.h_conv4_2, self.W_conv4_3, "conv4_2") + self.B_conv4_3) self.h_pool4 = maxpool_2x2(self.h_conv4_3, "pool4") with tf.name_scope("Conv5Ops"): self.W_conv5_1 = weight_variable_fromnp( npWeights["conv5_1_w"], "w_conv5_1") self.B_conv5_1 = weight_variable_fromnp( npWeights["conv5_1_b"], "b_conv5_1") self.W_conv5_2 = weight_variable_fromnp( npWeights["conv5_2_w"], "w_conv5_2") self.B_conv5_2 = weight_variable_fromnp( npWeights["conv5_2_b"], "b_conv5_2") self.W_conv5_3 = weight_variable_fromnp( npWeights["conv5_3_w"], "w_conv5_3") self.B_conv5_3 = weight_variable_fromnp( npWeights["conv5_3_b"], "b_conv5_3") self.h_conv5_1 = tf.nn.relu( conv2d(self.h_pool4, self.W_conv5_1, "conv5_1") + self.B_conv5_1) self.h_conv5_2 = tf.nn.relu( conv2d(self.h_conv5_1, self.W_conv5_2, "conv5_2") + self.B_conv5_2) self.h_conv5_3 = tf.nn.relu( conv2d(self.h_conv5_2, self.W_conv5_3, "conv5_2") + self.B_conv5_3) self.h_pool5 = maxpool_2x2(self.h_conv5_3, "pool5") #with tf.device('cpu:0'): self.keep_prob = tf.placeholder(tf.float32) with tf.name_scope("FC6"): self.W_fc6 = weight_variable_fromnp(npWeights["fc6_w"], "w_fc6") self.B_fc6 = weight_variable_fromnp(npWeights["fc6_b"], "b_fc6") h_pool5_flat = tf.reshape(self.h_pool5, [self.batchSize, 7 * 7 * 512]) self.h_fc6 = tf.nn.relu( tf.matmul(h_pool5_flat, self.W_fc6, name="fc6") + self.B_fc6, "fc6_relu") self.drop_h_fc6 = tf.nn.dropout(self.h_fc6, self.keep_prob) #with tf.device(self.device): with tf.name_scope("FC7"): self.W_fc7 = weight_variable_fromnp(npWeights["fc7_w"], "w_fc7") self.B_fc7 = weight_variable_fromnp(npWeights["fc7_b"], "b_fc7") self.h_fc7 = tf.nn.relu( tf.matmul(self.drop_h_fc6, self.W_fc7, name="fc7") + self.B_fc7, "fc7_relu") self.drop_h_fc7 = tf.nn.dropout(self.h_fc7, self.keep_prob) with tf.name_scope("FC8"): self.W_fc8 = weight_variable_xavier([4096, 20], "w_fc8") self.B_fc8 = bias_variable([20], "b_fc8") #self.est = tf.nn.softmax(tf.matmul(self.drop_h_fc7, self.W_fc8, name="fc8") + self.B_fc8) self.est = tf.matmul(self.drop_h_fc7, self.W_fc8, name="fc8") + self.B_fc8 with tf.name_scope("Loss"): #Grab positive and negative classes as needed self.posArray = tf.expand_dims(self.gt * self.est, dim=1) self.negArray = tf.expand_dims(self.notGt * self.est, dim=2) #tf broadcasting should take care of this to make this a [batch, class, class] matrix self.cSum = (1 - self.posArray) + self.negArray self.pairLoss = tf.reduce_mean(self.cSum, reduction_indices=[1, 2]) self.loss = tf.reduce_mean(tf.nn.relu(self.pairLoss)) #self.loss = tf.reduce_mean(-tf.reduce_sum(self.gt * tf.log(self.est+self.epsilon), reduction_indices=[1])) #self.regLoss = self.loss + self.regStrength * tf.add_n([tf.nn.l2_loss(v) for v in tf.trainable_variables()]) #self.nan_check_loss = tf.verify_tensor_all_finite(self.loss, msg="check_nan") with tf.name_scope("Opt"): #Define optimizer self.optimizerAll = tf.train.AdamOptimizer( self.learningRate, beta1=self.beta1, beta2=self.beta2, epsilon=self.epsilon).minimize(self.loss) #self.optimizerAll = tf.train.MomentumOptimizer(self.learningRate, momentum=self.beta1).minimize(self.loss) self.optimizerPre = tf.train.AdamOptimizer( self.learningRate, beta1=self.beta1, beta2=self.beta2, epsilon=self.epsilon).minimize(self.loss, var_list=[ self.W_fc6, self.B_fc6, self.W_fc7, self.B_fc7, self.W_fc8, self.B_fc8, ]) with tf.name_scope("Metric"): self.correct = tf.equal(tf.argmax(self.gt, 1), tf.argmax(self.est, 1)) self.accuracy = tf.reduce_mean( tf.cast(self.correct, tf.float32)) #Cannot be on GPU (self.eval_vals, self.eval_idx) = tf.nn.top_k(self.est, k=5) #Summaries tf.scalar_summary('loss', self.loss, name="lossSum") tf.scalar_summary('accuracy', self.accuracy, name="accSum") tf.histogram_summary('input', self.inputImage, name="image_vis") tf.histogram_summary('gt', self.gt, name="gt_vis") tf.histogram_summary('notGt', self.notGt, name="notGt_vis") #Conv layer histograms tf.histogram_summary('conv1_1', self.h_conv1_1, name="conv1_1_vis") tf.histogram_summary('conv1_2', self.h_conv1_2, name="conv1_2_vis") tf.histogram_summary('conv2_1', self.h_conv2_1, name="conv2_1_vis") tf.histogram_summary('conv2_2', self.h_conv2_2, name="conv2_2_vis") tf.histogram_summary('conv3_1', self.h_conv3_1, name="conv3_1_vis") tf.histogram_summary('conv3_2', self.h_conv3_2, name="conv3_2_vis") tf.histogram_summary('conv3_3', self.h_conv3_3, name="conv3_3_vis") tf.histogram_summary('conv4_1', self.h_conv4_1, name="conv4_1_vis") tf.histogram_summary('conv4_2', self.h_conv4_2, name="conv4_2_vis") tf.histogram_summary('conv4_3', self.h_conv4_3, name="conv4_3_vis") tf.histogram_summary('conv5_1', self.h_conv5_1, name="conv5_1_vis") tf.histogram_summary('conv5_2', self.h_conv5_2, name="conv5_2_vis") tf.histogram_summary('conv5_3', self.h_conv5_3, name="conv5_3_vis") tf.histogram_summary('fc6', self.h_fc6, name="fc6_vis") tf.histogram_summary('fc7', self.h_fc7, name="fc7_vis") tf.histogram_summary('est', self.est, name="est_vis") tf.histogram_summary('posArray', self.posArray, name="est_vis") tf.histogram_summary('negArray', self.negArray, name="est_vis") tf.histogram_summary('cSum', self.cSum, name="est_vis") tf.histogram_summary('pairLoss', self.pairLoss, name="est_vis") #Weight and bias hists tf.histogram_summary('w_conv1_1', self.W_conv1_1, name="w_conv1_1_vis") tf.histogram_summary('b_conv1_1', self.B_conv1_1, name="b_conv1_1_vis") tf.histogram_summary('w_conv1_2', self.W_conv1_2, name="w_conv1_2_vis") tf.histogram_summary('b_conv1_2', self.B_conv1_2, name="b_conv1_2_vis") tf.histogram_summary('w_conv2_1', self.W_conv2_1, name="w_conv2_1_vis") tf.histogram_summary('b_conv2_1', self.B_conv2_1, name="b_conv2_1_vis") tf.histogram_summary('w_conv2_2', self.W_conv2_2, name="w_conv2_2_vis") tf.histogram_summary('b_conv2_2', self.B_conv2_2, name="b_conv2_2_vis") tf.histogram_summary('w_conv3_1', self.W_conv3_1, name="w_conv3_1_vis") tf.histogram_summary('b_conv3_1', self.B_conv3_1, name="b_conv3_1_vis") tf.histogram_summary('w_conv3_2', self.W_conv3_2, name="w_conv3_2_vis") tf.histogram_summary('b_conv3_2', self.B_conv3_2, name="b_conv3_2_vis") tf.histogram_summary('w_conv3_3', self.W_conv3_3, name="w_conv3_3_vis") tf.histogram_summary('b_conv3_3', self.B_conv3_3, name="b_conv3_3_vis") tf.histogram_summary('w_conv4_1', self.W_conv4_1, name="w_conv4_1_vis") tf.histogram_summary('b_conv4_1', self.B_conv4_1, name="b_conv4_1_vis") tf.histogram_summary('w_conv4_2', self.W_conv4_2, name="w_conv4_2_vis") tf.histogram_summary('b_conv4_2', self.B_conv4_2, name="b_conv4_2_vis") tf.histogram_summary('w_conv4_3', self.W_conv4_3, name="w_conv4_3_vis") tf.histogram_summary('b_conv4_3', self.B_conv4_3, name="b_conv4_3_vis") tf.histogram_summary('w_conv5_1', self.W_conv5_1, name="w_conv5_1_vis") tf.histogram_summary('b_conv5_1', self.B_conv5_1, name="b_conv5_1_vis") tf.histogram_summary('w_conv5_2', self.W_conv5_2, name="w_conv5_2_vis") tf.histogram_summary('b_conv5_2', self.B_conv5_2, name="b_conv5_2_vis") tf.histogram_summary('w_conv5_3', self.W_conv5_3, name="w_conv5_3_vis") tf.histogram_summary('b_conv5_3', self.B_conv5_3, name="b_conv5_3_vis") tf.histogram_summary('w_fc6', self.W_fc6, name="w_fc6_vis") tf.histogram_summary('b_fc6', self.B_fc6, name="b_fc6_vis") tf.histogram_summary('w_fc7', self.W_fc7, name="w_fc7_vis") tf.histogram_summary('b_fc7', self.B_fc7, name="b_fc7_vis") tf.histogram_summary('w_fc8', self.W_fc7, name="w_fc8_vis") tf.histogram_summary('b_fc8', self.B_fc7, name="b_fc8_vis")
def buildModel(self, inputShape, inMatFilename): if(inMatFilename): npWeights = loadWeights(inMatFilename) #Running on GPU with tf.device('gpu:0'): with tf.name_scope("inputOps"): #Get convolution variables as placeholders self.inputImage = node_variable([None, inputShape[0], inputShape[1], inputShape[2]], "inputImage") self.gt = node_variable([None, 2], "gt") #Model variables for convolutions with tf.name_scope("Conv1Ops"): if(inMatFilename): self.W_conv1 = weight_variable_fromnp(npWeights["conv1_w"], "w_conv1") self.B_conv1 = weight_variable_fromnp(npWeights["conv1_b"], "b_conv1") else: self.W_conv1 = weight_variable_fromnp(np.zeros((11, 11, 3, 64), dtype=np.float32), "w_conv1") self.B_conv1 = weight_variable_fromnp(np.zeros((64), dtype=np.float32), "b_conv1") #self.W_conv1 = weight_variable_xavier([11, 11, 3, 64], "w_conv1", conv=True) #self.B_conv1 = bias_variable([64], "b_conv1") self.h_conv1 = tf.nn.relu(conv2d(self.inputImage, self.W_conv1, "conv1", stride=[1, 4, 4, 1]) + self.B_conv1) self.h_norm1 = tf.nn.local_response_normalization(self.h_conv1, name="LRN1") self.h_pool1 = maxpool_2x2(self.h_norm1, "pool1") with tf.name_scope("Conv2Ops"): if(inMatFilename): self.W_conv2 = weight_variable_fromnp(npWeights["conv2_w"], "w_conv2") self.B_conv2 = weight_variable_fromnp(npWeights["conv2_b"], "b_conv2") else: self.W_conv2 = weight_variable_fromnp(np.zeros((5, 5, 64, 256), dtype=np.float32), "w_conv2") self.B_conv2 = weight_variable_fromnp(np.zeros((256), dtype=np.float32), "b_conv2") #self.W_conv2 = weight_variable_xavier([5, 5, 64, 256], "w_conv2", conv=True) #self.B_conv2 = bias_variable([256], "b_conv2") self.h_conv2 = tf.nn.relu(conv2d(self.h_pool1, self.W_conv2, "conv2") + self.B_conv2) self.h_norm2 = tf.nn.local_response_normalization(self.h_conv2, name="LRN2") self.h_pool2 = maxpool_2x2(self.h_norm2, "pool2") with tf.name_scope("Conv3Ops"): if(inMatFilename): self.W_conv3 = weight_variable_fromnp(npWeights["conv3_w"], "w_conv3") self.B_conv3 = weight_variable_fromnp(npWeights["conv3_b"], "b_conv3") else: self.W_conv3 = weight_variable_fromnp(np.zeros((3, 3, 256, 256), dtype=np.float32), "w_conv3") self.B_conv3 = weight_variable_fromnp(np.zeros((256), dtype=np.float32), "b_conv3") #self.W_conv3 = weight_variable_xavier([3, 3, 256, 256], "w_conv3", conv=True) #self.B_conv3 = bias_variable([256], "b_conv3") self.h_conv3 = tf.nn.relu(conv2d(self.h_pool2, self.W_conv3, "conv3") + self.B_conv3, name="relu3") with tf.name_scope("Conv4Ops"): if(inMatFilename): self.W_conv4 = weight_variable_fromnp(npWeights["conv4_w"], "w_conv4") self.B_conv4 = weight_variable_fromnp(npWeights["conv4_b"], "b_conv4") else: self.W_conv4 = weight_variable_fromnp(np.zeros((3, 3, 256, 256), dtype=np.float32), "w_conv4") self.B_conv4 = weight_variable_fromnp(np.zeros((256), dtype=np.float32), "b_conv4") #self.W_conv4 = weight_variable_xavier([3, 3, 256, 256], "w_conv4", conv=True) #self.B_conv4 = bias_variable([256], "b_conv4") self.h_conv4 = tf.nn.relu(conv2d(self.h_conv3, self.W_conv4, "conv4") + self.B_conv4, name="relu4") with tf.name_scope("Conv5Ops"): if(inMatFilename): self.W_conv5 = weight_variable_fromnp(npWeights["conv5_w"], "w_conv5") self.B_conv5 = weight_variable_fromnp(npWeights["conv5_b"], "b_conv5") else: self.W_conv5 = weight_variable_fromnp(np.zeros((3, 3, 256, 256), dtype=np.float32), "w_conv5") self.B_conv5 = weight_variable_fromnp(np.zeros((256), dtype = np.float32), "b_conv5") #self.W_conv5 = weight_variable_xavier([3, 3, 256, 256], "w_conv5", conv=True) #self.B_conv5 = bias_variable([256], "b_conv5") self.h_conv5 = tf.nn.relu(conv2d(self.h_conv4, self.W_conv5, "conv5") + self.B_conv5) self.h_pool5 = maxpool_2x2(self.h_conv5, "pool5") #placeholder for specifying dropout self.keep_prob = tf.placeholder(tf.float32) #32 comes from 4 stride in conv1, 2 stride in pool1, 2 stride in pool2, 2 stride in pool5 numInputs = (inputShape[0]/32) * (inputShape[1]/32) * 256 with tf.name_scope("FC1"): #if(inMatFilename): # self.W_conv5 = weight_variable_fromnp(npWeights["fc1_w"], "w_fc1") # self.B_conv5 = weight_variable_fromnp(npWeights["fc1_b"], "b_fc1") #else: # self.W_conv5 = weight_variable_fromnp(np.zeros((6*6*256, 4096), dtype=np.float32), "w_fc1") # self.B_conv5 = weight_variable_fromnp(np.zeros((4096), dtype = np.float32), "b_fc1") self.W_fc1 = weight_variable_xavier([numInputs, 4096], "w_fc1") self.B_fc1 = bias_variable([4096], "b_fc1") h_pool5_flat = tf.reshape(self.h_pool5, [-1, numInputs], name="pool5_flat") self.h_fc1 = tf.nn.relu(tf.matmul(h_pool5_flat, self.W_fc1, name="fc1") + self.B_fc1, "fc1_relu") self.h_fc1_drop = tf.nn.dropout(self.h_fc1, self.keep_prob) with tf.name_scope("FC2"): #if(inMatFilename): # self.W_conv5 = weight_variable_fromnp(npWeights["fc2_w"], "w_fc2") # self.B_conv5 = weight_variable_fromnp(npWeights["fc2_b"], "b_fc2") #else: # self.W_conv5 = weight_variable_fromnp(np.zeros((4096, 4096), dtype=np.float32), "w_fc2") # self.B_conv5 = weight_variable_fromnp(np.zeros((4096), dtype = np.float32), "b_fc2") self.W_fc2 = weight_variable_xavier([4096, 4096], "w_fc2") self.B_fc2 = bias_variable([4096], "b_fc2") self.h_fc2 = tf.nn.relu(tf.matmul(self.h_fc1_drop, self.W_fc2, name="fc2") + self.B_fc2, "fc2_relu") self.h_fc2_drop = tf.nn.dropout(self.h_fc2, self.keep_prob) #fc3 should have 16 channels #fc3 also uses a sigmoid function #We change it to tanh with tf.name_scope("FC3"): #if(inMatFilename): # self.W_conv5 = weight_variable_fromnp(npWeights["fc3_w"], "w_fc3") # self.B_conv5 = weight_variable_fromnp(npWeights["fc3_b"], "b_fc3") #else: # self.W_conv5 = weight_variable_fromnp(np.zeros((4096, 2), dtype=np.float32), "w_fc3") # self.B_conv5 = weight_variable_fromnp(np.zeros((2), dtype = np.float32), "b_fc3") self.W_fc3 = weight_variable_xavier([4096, 2], "w_fc3") self.B_fc3 = bias_variable([2], "b_fc3") self.est = tf.nn.softmax(tf.matmul(self.h_fc2_drop, self.W_fc3, name="fc3") + self.B_fc3, "fc3_softmax") with tf.name_scope("Loss"): #Define loss #self.loss = tf.reduce_mean(-tf.reduce_sum(self.gt * tf.log(self.est), reduction_indices=[1])) self.loss = tf.reduce_mean(-(self.gt[:, 1]*.8* tf.log(self.est[:, 1]) + self.gt[:, 0]*.2*tf.log(self.est[:, 0]))) with tf.name_scope("Opt"): #Define optimizer #self.optimizerAll = tf.train.AdagradOptimizer(self.learningRate).minimize(self.loss) #self.optimizerFC = tf.train.AdagradOptimizer(self.learningRate).minimize(self.loss, self.optimizerAll = tf.train.AdamOptimizer(self.learningRate).minimize(self.loss) self.optimizerFC = tf.train.AdamOptimizer(self.learningRate).minimize(self.loss, var_list=[ self.W_fc1, self.B_fc1, self.W_fc2, self.B_fc2, self.W_fc3, self.B_fc3] ) with tf.name_scope("Metric"): self.gtIdx = tf.argmax(self.gt, 1) self.estIdx = tf.argmax(self.est, 1) boolGtIdx = tf.cast(self.gtIdx, tf.bool) boolEstIdx = tf.cast(self.estIdx, tf.bool) #Logical and for true positive lAnd = tf.logical_and(boolGtIdx, boolEstIdx) self.tp = tf.reduce_sum(tf.cast(lAnd, tf.float32)) #Logical nor for true negatives lNor = tf.logical_not(tf.logical_or(boolGtIdx, boolEstIdx)) self.tn = tf.reduce_sum(tf.cast(lNor, tf.float32)) #Subtraction and comparison for others lSub = self.gtIdx - self.estIdx Ones = tf.cast(tf.ones(tf.shape(lSub)), tf.int64) self.fn = tf.reduce_sum(tf.cast(tf.equal(lSub, Ones), tf.float32)) self.fp = tf.reduce_sum(tf.cast(tf.equal(lSub, -Ones), tf.float32)) #Accuracy, precision, and recall calculations self.accuracy = (self.tp + self.tn)/(self.tp+self.tn+self.fp+self.fn) self.precision = self.tp/(self.tp+self.fp) self.recall = self.tp/(self.tp+self.fn) #Summaries tf.scalar_summary('loss', self.loss, name="lossSum") tf.scalar_summary('accuracy', self.accuracy, name="accSum") tf.scalar_summary('precision', self.precision, name="precSum") tf.scalar_summary('recall', self.recall, name="recallSum") tf.scalar_summary('tp', self.tp, name="tp") tf.scalar_summary('fp', self.fp, name="fp") tf.scalar_summary('tn', self.tn, name="tn") tf.scalar_summary('fn', self.fn, name="fn") tf.histogram_summary('input', self.inputImage, name="image") tf.histogram_summary('gt', self.gt, name="gt") tf.histogram_summary('conv1', self.h_pool1, name="conv1") tf.histogram_summary('conv2', self.h_pool2, name="conv2") tf.histogram_summary('conv3', self.h_conv3, name="conv3") tf.histogram_summary('conv4', self.h_conv4, name="conv4") tf.histogram_summary('conv5', self.h_pool5, name="conv5") tf.histogram_summary('fc1', self.h_fc1, name="fc1") tf.histogram_summary('fc2', self.h_fc2, name="fc2") tf.histogram_summary('est', self.est, name="fc3") tf.histogram_summary('w_conv1', self.W_conv1, name="w_conv1") tf.histogram_summary('b_conv1', self.B_conv1, name="b_conv1") tf.histogram_summary('w_conv2', self.W_conv2, name="w_conv2") tf.histogram_summary('b_conv2', self.B_conv2, name="b_conv2") tf.histogram_summary('w_conv3', self.W_conv3, name="w_conv3") tf.histogram_summary('b_conv3', self.B_conv3, name="b_conv3") tf.histogram_summary('w_conv4', self.W_conv4, name="w_conv4") tf.histogram_summary('b_conv4', self.B_conv4, name="b_conv4") tf.histogram_summary('w_conv5', self.W_conv5, name="w_conv5") tf.histogram_summary('b_conv5', self.B_conv5, name="b_conv5") tf.histogram_summary('w_fc1', self.W_fc1, name="w_fc1") tf.histogram_summary('b_fc1', self.B_fc1, name="b_fc1") tf.histogram_summary('w_fc2', self.W_fc2, name="w_fc2") tf.histogram_summary('b_fc2', self.B_fc2, name="b_fc2") tf.histogram_summary('w_fc3', self.W_fc3, name="w_fc3") tf.histogram_summary('b_fc3', self.B_fc3, name="b_fc3") #Define saver self.saver = tf.train.Saver()
def buildModel(self, inputShape): if(self.vggFile): npWeights = loadWeights(self.vggFile) else: print "Must load from weights" assert(0) #Running on GPU with tf.device(self.device): with tf.name_scope("inputOps"): #Get convolution variables as placeholders self.inputImage = node_variable([self.batchSize, inputShape[0], inputShape[1], inputShape[2]], "inputImage") self.gt = node_variable([self.batchSize, self.numClasses], "gt") self.notGt = 1-self.gt with tf.name_scope("Conv1Ops"): self.W_conv1_1 = weight_variable_fromnp(npWeights["conv1_1_w"], "w_conv1_1") self.B_conv1_1 = weight_variable_fromnp(npWeights["conv1_1_b"], "b_conv1_1") self.W_conv1_2 = weight_variable_fromnp(npWeights["conv1_2_w"], "w_conv1_2") self.B_conv1_2 = weight_variable_fromnp(npWeights["conv1_2_b"], "b_conv1_2") self.h_conv1_1 = tf.nn.relu(conv2d(self.inputImage, self.W_conv1_1, "conv1_1", stride=[1, 1, 1, 1]) + self.B_conv1_1) self.h_conv1_2 = tf.nn.relu(conv2d(self.h_conv1_1, self.W_conv1_2, "conv1_1", stride=[1, 1, 1, 1]) + self.B_conv1_2) self.h_pool1 = maxpool_2x2(self.h_conv1_2, "pool1") with tf.name_scope("Conv2Ops"): self.W_conv2_1 = weight_variable_fromnp(npWeights["conv2_1_w"], "w_conv2_1") self.B_conv2_1 = weight_variable_fromnp(npWeights["conv2_1_b"], "b_conv2_1") self.W_conv2_2 = weight_variable_fromnp(npWeights["conv2_2_w"], "w_conv2_2") self.B_conv2_2 = weight_variable_fromnp(npWeights["conv2_2_b"], "b_conv2_2") self.h_conv2_1 = tf.nn.relu(conv2d(self.h_pool1, self.W_conv2_1, "conv2_1") + self.B_conv2_1) self.h_conv2_2 = tf.nn.relu(conv2d(self.h_conv2_1, self.W_conv2_2, "conv2_2") + self.B_conv2_2) self.h_pool2 = maxpool_2x2(self.h_conv2_2, "pool2") with tf.name_scope("Conv3Ops"): self.W_conv3_1 = weight_variable_fromnp(npWeights["conv3_1_w"], "w_conv3_1") self.B_conv3_1 = weight_variable_fromnp(npWeights["conv3_1_b"], "b_conv3_1") self.W_conv3_2 = weight_variable_fromnp(npWeights["conv3_2_w"], "w_conv3_2") self.B_conv3_2 = weight_variable_fromnp(npWeights["conv3_2_b"], "b_conv3_2") self.W_conv3_3 = weight_variable_fromnp(npWeights["conv3_3_w"], "w_conv3_3") self.B_conv3_3 = weight_variable_fromnp(npWeights["conv3_3_b"], "b_conv3_3") self.h_conv3_1 = tf.nn.relu(conv2d(self.h_pool2, self.W_conv3_1, "conv3_1") + self.B_conv3_1) self.h_conv3_2 = tf.nn.relu(conv2d(self.h_conv3_1, self.W_conv3_2, "conv3_2") + self.B_conv3_2) self.h_conv3_3 = tf.nn.relu(conv2d(self.h_conv3_2, self.W_conv3_3, "conv3_2") + self.B_conv3_3) self.h_pool3 = maxpool_2x2(self.h_conv3_3, "pool3") with tf.name_scope("Conv4Ops"): self.W_conv4_1 = weight_variable_fromnp(npWeights["conv4_1_w"], "w_conv4_1") self.B_conv4_1 = weight_variable_fromnp(npWeights["conv4_1_b"], "b_conv4_1") self.W_conv4_2 = weight_variable_fromnp(npWeights["conv4_2_w"], "w_conv4_2") self.B_conv4_2 = weight_variable_fromnp(npWeights["conv4_2_b"], "b_conv4_2") self.W_conv4_3 = weight_variable_fromnp(npWeights["conv4_3_w"], "w_conv4_3") self.B_conv4_3 = weight_variable_fromnp(npWeights["conv4_3_b"], "b_conv4_3") self.h_conv4_1 = tf.nn.relu(conv2d(self.h_pool3, self.W_conv4_1, "conv4_1") + self.B_conv4_1) self.h_conv4_2 = tf.nn.relu(conv2d(self.h_conv4_1, self.W_conv4_2, "conv4_2") + self.B_conv4_2) self.h_conv4_3 = tf.nn.relu(conv2d(self.h_conv4_2, self.W_conv4_3, "conv4_2") + self.B_conv4_3) self.h_pool4 = maxpool_2x2(self.h_conv4_3, "pool4") with tf.name_scope("Conv5Ops"): self.W_conv5_1 = weight_variable_fromnp(npWeights["conv5_1_w"], "w_conv5_1") self.B_conv5_1 = weight_variable_fromnp(npWeights["conv5_1_b"], "b_conv5_1") self.W_conv5_2 = weight_variable_fromnp(npWeights["conv5_2_w"], "w_conv5_2") self.B_conv5_2 = weight_variable_fromnp(npWeights["conv5_2_b"], "b_conv5_2") self.W_conv5_3 = weight_variable_fromnp(npWeights["conv5_3_w"], "w_conv5_3") self.B_conv5_3 = weight_variable_fromnp(npWeights["conv5_3_b"], "b_conv5_3") self.h_conv5_1 = tf.nn.relu(conv2d(self.h_pool4, self.W_conv5_1, "conv5_1") + self.B_conv5_1) self.h_conv5_2 = tf.nn.relu(conv2d(self.h_conv5_1, self.W_conv5_2, "conv5_2") + self.B_conv5_2) self.h_conv5_3 = tf.nn.relu(conv2d(self.h_conv5_2, self.W_conv5_3, "conv5_2") + self.B_conv5_3) self.h_pool5 = maxpool_2x2(self.h_conv5_3, "pool5") #with tf.device('cpu:0'): self.keep_prob = tf.placeholder(tf.float32) with tf.name_scope("FC6"): self.W_fc6 = weight_variable_fromnp(npWeights["fc6_w"], "w_fc6") self.B_fc6 = weight_variable_fromnp(npWeights["fc6_b"], "b_fc6") h_pool5_flat = tf.reshape(self.h_pool5, [self.batchSize, 7*7*512]) self.h_fc6 = tf.nn.relu(tf.matmul(h_pool5_flat, self.W_fc6, name="fc6") + self.B_fc6, "fc6_relu") self.drop_h_fc6 = tf.nn.dropout(self.h_fc6, self.keep_prob) #with tf.device(self.device): with tf.name_scope("FC7"): self.W_fc7 = weight_variable_fromnp(npWeights["fc7_w"], "w_fc7") self.B_fc7 = weight_variable_fromnp(npWeights["fc7_b"], "b_fc7") self.h_fc7 = tf.nn.relu(tf.matmul(self.drop_h_fc6, self.W_fc7, name="fc7") + self.B_fc7, "fc7_relu") self.drop_h_fc7 = tf.nn.dropout(self.h_fc7, self.keep_prob) with tf.name_scope("FC8"): self.W_fc8 = weight_variable_xavier([4096, 20], "w_fc8") self.B_fc8 = bias_variable([20], "b_fc8") #self.est = tf.nn.softmax(tf.matmul(self.drop_h_fc7, self.W_fc8, name="fc8") + self.B_fc8) self.est = tf.matmul(self.drop_h_fc7, self.W_fc8, name="fc8") + self.B_fc8 with tf.name_scope("Loss"): #Grab positive and negative classes as needed self.posArray = tf.expand_dims(self.gt*self.est, dim=1) self.negArray = tf.expand_dims(self.notGt*self.est, dim=2) #tf broadcasting should take care of this to make this a [batch, class, class] matrix self.cSum = (1 - self.posArray) + self.negArray self.pairLoss = tf.reduce_mean(self.cSum, reduction_indices=[1, 2]) self.loss = tf.reduce_mean(tf.nn.relu(self.pairLoss)) #self.loss = tf.reduce_mean(-tf.reduce_sum(self.gt * tf.log(self.est+self.epsilon), reduction_indices=[1])) #self.regLoss = self.loss + self.regStrength * tf.add_n([tf.nn.l2_loss(v) for v in tf.trainable_variables()]) #self.nan_check_loss = tf.verify_tensor_all_finite(self.loss, msg="check_nan") with tf.name_scope("Opt"): #Define optimizer self.optimizerAll = tf.train.AdamOptimizer(self.learningRate, beta1=self.beta1, beta2=self.beta2, epsilon=self.epsilon).minimize(self.loss) #self.optimizerAll = tf.train.MomentumOptimizer(self.learningRate, momentum=self.beta1).minimize(self.loss) self.optimizerPre = tf.train.AdamOptimizer(self.learningRate, beta1=self.beta1, beta2=self.beta2, epsilon=self.epsilon).minimize(self.loss, var_list=[ self.W_fc6, self.B_fc6, self.W_fc7, self.B_fc7, self.W_fc8, self.B_fc8, ] ) with tf.name_scope("Metric"): self.correct = tf.equal(tf.argmax(self.gt, 1), tf.argmax(self.est, 1)) self.accuracy = tf.reduce_mean(tf.cast(self.correct, tf.float32)) #Cannot be on GPU (self.eval_vals, self.eval_idx) = tf.nn.top_k(self.est, k=5) #Summaries tf.scalar_summary('loss', self.loss, name="lossSum") tf.scalar_summary('accuracy', self.accuracy, name="accSum") tf.histogram_summary('input', self.inputImage, name="image_vis") tf.histogram_summary('gt', self.gt, name="gt_vis") tf.histogram_summary('notGt', self.notGt, name="notGt_vis") #Conv layer histograms tf.histogram_summary('conv1_1', self.h_conv1_1, name="conv1_1_vis") tf.histogram_summary('conv1_2', self.h_conv1_2, name="conv1_2_vis") tf.histogram_summary('conv2_1', self.h_conv2_1, name="conv2_1_vis") tf.histogram_summary('conv2_2', self.h_conv2_2, name="conv2_2_vis") tf.histogram_summary('conv3_1', self.h_conv3_1, name="conv3_1_vis") tf.histogram_summary('conv3_2', self.h_conv3_2, name="conv3_2_vis") tf.histogram_summary('conv3_3', self.h_conv3_3, name="conv3_3_vis") tf.histogram_summary('conv4_1', self.h_conv4_1, name="conv4_1_vis") tf.histogram_summary('conv4_2', self.h_conv4_2, name="conv4_2_vis") tf.histogram_summary('conv4_3', self.h_conv4_3, name="conv4_3_vis") tf.histogram_summary('conv5_1', self.h_conv5_1, name="conv5_1_vis") tf.histogram_summary('conv5_2', self.h_conv5_2, name="conv5_2_vis") tf.histogram_summary('conv5_3', self.h_conv5_3, name="conv5_3_vis") tf.histogram_summary('fc6', self.h_fc6, name="fc6_vis") tf.histogram_summary('fc7', self.h_fc7, name="fc7_vis") tf.histogram_summary('est', self.est, name="est_vis") tf.histogram_summary('posArray', self.posArray, name="est_vis") tf.histogram_summary('negArray', self.negArray, name="est_vis") tf.histogram_summary('cSum', self.cSum, name="est_vis") tf.histogram_summary('pairLoss', self.pairLoss, name="est_vis") #Weight and bias hists tf.histogram_summary('w_conv1_1', self.W_conv1_1, name="w_conv1_1_vis") tf.histogram_summary('b_conv1_1', self.B_conv1_1, name="b_conv1_1_vis") tf.histogram_summary('w_conv1_2', self.W_conv1_2, name="w_conv1_2_vis") tf.histogram_summary('b_conv1_2', self.B_conv1_2, name="b_conv1_2_vis") tf.histogram_summary('w_conv2_1', self.W_conv2_1, name="w_conv2_1_vis") tf.histogram_summary('b_conv2_1', self.B_conv2_1, name="b_conv2_1_vis") tf.histogram_summary('w_conv2_2', self.W_conv2_2, name="w_conv2_2_vis") tf.histogram_summary('b_conv2_2', self.B_conv2_2, name="b_conv2_2_vis") tf.histogram_summary('w_conv3_1', self.W_conv3_1, name="w_conv3_1_vis") tf.histogram_summary('b_conv3_1', self.B_conv3_1, name="b_conv3_1_vis") tf.histogram_summary('w_conv3_2', self.W_conv3_2, name="w_conv3_2_vis") tf.histogram_summary('b_conv3_2', self.B_conv3_2, name="b_conv3_2_vis") tf.histogram_summary('w_conv3_3', self.W_conv3_3, name="w_conv3_3_vis") tf.histogram_summary('b_conv3_3', self.B_conv3_3, name="b_conv3_3_vis") tf.histogram_summary('w_conv4_1', self.W_conv4_1, name="w_conv4_1_vis") tf.histogram_summary('b_conv4_1', self.B_conv4_1, name="b_conv4_1_vis") tf.histogram_summary('w_conv4_2', self.W_conv4_2, name="w_conv4_2_vis") tf.histogram_summary('b_conv4_2', self.B_conv4_2, name="b_conv4_2_vis") tf.histogram_summary('w_conv4_3', self.W_conv4_3, name="w_conv4_3_vis") tf.histogram_summary('b_conv4_3', self.B_conv4_3, name="b_conv4_3_vis") tf.histogram_summary('w_conv5_1', self.W_conv5_1, name="w_conv5_1_vis") tf.histogram_summary('b_conv5_1', self.B_conv5_1, name="b_conv5_1_vis") tf.histogram_summary('w_conv5_2', self.W_conv5_2, name="w_conv5_2_vis") tf.histogram_summary('b_conv5_2', self.B_conv5_2, name="b_conv5_2_vis") tf.histogram_summary('w_conv5_3', self.W_conv5_3, name="w_conv5_3_vis") tf.histogram_summary('b_conv5_3', self.B_conv5_3, name="b_conv5_3_vis") tf.histogram_summary('w_fc6', self.W_fc6, name="w_fc6_vis") tf.histogram_summary('b_fc6', self.B_fc6, name="b_fc6_vis") tf.histogram_summary('w_fc7', self.W_fc7, name="w_fc7_vis") tf.histogram_summary('b_fc7', self.B_fc7, name="b_fc7_vis") tf.histogram_summary('w_fc8', self.W_fc7, name="w_fc8_vis") tf.histogram_summary('b_fc8', self.B_fc7, name="b_fc8_vis")
def buildModel(self, inputShape): if (self.vggFile): npWeights = loadWeights(self.vggFile) else: print "Must load from weights" assert (0) #Running on GPU with tf.device(self.device): with tf.name_scope("inputOps"): #Get convolution variables as placeholders self.inputImage = node_variable([ self.batchSize, inputShape[0], inputShape[1], inputShape[2] ], "inputImage") self.gt = node_variable( [self.batchSize, 7, 7, self.numClasses], "gt") self.norm_gt = self.gt / tf.reduce_sum( self.gt, reduction_indices=3, keep_dims=True) #Model variables for convolutions with tf.name_scope("Conv1Ops"): self.W_conv1_1 = weight_variable_fromnp( npWeights["conv1_1_w"], "w_conv1_1") self.B_conv1_1 = weight_variable_fromnp( npWeights["conv1_1_b"], "b_conv1_1") self.W_conv1_2 = weight_variable_fromnp( npWeights["conv1_2_w"], "w_conv1_2") self.B_conv1_2 = weight_variable_fromnp( npWeights["conv1_2_b"], "b_conv1_2") self.h_conv1_1 = tf.nn.relu( conv2d(self.inputImage, self.W_conv1_1, "conv1_1", stride=[1, 1, 1, 1]) + self.B_conv1_1) self.h_conv1_2 = tf.nn.relu( conv2d(self.h_conv1_1, self.W_conv1_2, "conv1_1", stride=[1, 1, 1, 1]) + self.B_conv1_2) self.h_pool1 = maxpool_2x2(self.h_conv1_2, "pool1") with tf.name_scope("Conv2Ops"): self.W_conv2_1 = weight_variable_fromnp( npWeights["conv2_1_w"], "w_conv2_1") self.B_conv2_1 = weight_variable_fromnp( npWeights["conv2_1_b"], "b_conv2_1") self.W_conv2_2 = weight_variable_fromnp( npWeights["conv2_2_w"], "w_conv2_2") self.B_conv2_2 = weight_variable_fromnp( npWeights["conv2_2_b"], "b_conv2_2") self.h_conv2_1 = tf.nn.relu( conv2d(self.h_pool1, self.W_conv2_1, "conv2_1") + self.B_conv2_1) self.h_conv2_2 = tf.nn.relu( conv2d(self.h_conv2_1, self.W_conv2_2, "conv2_2") + self.B_conv2_2) self.h_pool2 = maxpool_2x2(self.h_conv2_2, "pool2") with tf.name_scope("Conv3Ops"): self.W_conv3_1 = weight_variable_fromnp( npWeights["conv3_1_w"], "w_conv3_1") self.B_conv3_1 = weight_variable_fromnp( npWeights["conv3_1_b"], "b_conv3_1") self.W_conv3_2 = weight_variable_fromnp( npWeights["conv3_2_w"], "w_conv3_2") self.B_conv3_2 = weight_variable_fromnp( npWeights["conv3_2_b"], "b_conv3_2") self.W_conv3_3 = weight_variable_fromnp( npWeights["conv3_3_w"], "w_conv3_3") self.B_conv3_3 = weight_variable_fromnp( npWeights["conv3_3_b"], "b_conv3_3") self.h_conv3_1 = tf.nn.relu( conv2d(self.h_pool2, self.W_conv3_1, "conv3_1") + self.B_conv3_1) self.h_conv3_2 = tf.nn.relu( conv2d(self.h_conv3_1, self.W_conv3_2, "conv3_2") + self.B_conv3_2) self.h_conv3_3 = tf.nn.relu( conv2d(self.h_conv3_2, self.W_conv3_3, "conv3_2") + self.B_conv3_3) self.h_pool3 = maxpool_2x2(self.h_conv3_3, "pool3") with tf.name_scope("Conv4Ops"): self.W_conv4_1 = weight_variable_fromnp( npWeights["conv4_1_w"], "w_conv4_1") self.B_conv4_1 = weight_variable_fromnp( npWeights["conv4_1_b"], "b_conv4_1") self.W_conv4_2 = weight_variable_fromnp( npWeights["conv4_2_w"], "w_conv4_2") self.B_conv4_2 = weight_variable_fromnp( npWeights["conv4_2_b"], "b_conv4_2") self.W_conv4_3 = weight_variable_fromnp( npWeights["conv4_3_w"], "w_conv4_3") self.B_conv4_3 = weight_variable_fromnp( npWeights["conv4_3_b"], "b_conv4_3") self.h_conv4_1 = tf.nn.relu( conv2d(self.h_pool3, self.W_conv4_1, "conv4_1") + self.B_conv4_1) self.h_conv4_2 = tf.nn.relu( conv2d(self.h_conv4_1, self.W_conv4_2, "conv4_2") + self.B_conv4_2) self.h_conv4_3 = tf.nn.relu( conv2d(self.h_conv4_2, self.W_conv4_3, "conv4_2") + self.B_conv4_3) self.h_pool4 = maxpool_2x2(self.h_conv4_3, "pool4") with tf.name_scope("Conv5Ops"): self.W_conv5_1 = weight_variable_fromnp( npWeights["conv5_1_w"], "w_conv5_1") self.B_conv5_1 = weight_variable_fromnp( npWeights["conv5_1_b"], "b_conv5_1") self.W_conv5_2 = weight_variable_fromnp( npWeights["conv5_2_w"], "w_conv5_2") self.B_conv5_2 = weight_variable_fromnp( npWeights["conv5_2_b"], "b_conv5_2") self.W_conv5_3 = weight_variable_fromnp( npWeights["conv5_3_w"], "w_conv5_3") self.B_conv5_3 = weight_variable_fromnp( npWeights["conv5_3_b"], "b_conv5_3") self.h_conv5_1 = tf.nn.relu( conv2d(self.h_pool4, self.W_conv5_1, "conv5_1") + self.B_conv5_1) self.h_conv5_2 = tf.nn.relu( conv2d(self.h_conv5_1, self.W_conv5_2, "conv5_2") + self.B_conv5_2) self.h_conv5_3 = tf.nn.relu( conv2d(self.h_conv5_2, self.W_conv5_3, "conv5_2") + self.B_conv5_3) self.h_pool5 = tf.nn.avg_pool(self.h_conv5_3, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME', name="pool5") #16 comes from 4 2x2 pooling #self.h_conv5_shape = [self.batchSize, inputShape[0]/32, inputShape[1]/32, 512] self.h_conv5_shape = [self.batchSize, 7, 7, 512] with tf.name_scope("GAP"): self.h_gap = tf.reduce_mean(self.h_pool5, reduction_indices=[1, 2]) self.W_gap = weight_variable_xavier([512, self.numClasses], "w_gap", conv=False) self.B_gap = bias_variable([self.numClasses], "b_gap") self.est = tf.nn.softmax( tf.matmul(self.h_gap, self.W_gap) + self.B_gap) with tf.name_scope("CAM"): self.h_reshape_gap = tf.reshape(self.h_pool5, [ self.batchSize * self.h_conv5_shape[1] * self.h_conv5_shape[2], -1 ]) self.flat_cam = tf.matmul(self.h_reshape_gap, self.W_gap) + self.B_gap self.reshape_cam = tf.reshape(self.flat_cam, [ self.batchSize, self.h_conv5_shape[1], self.h_conv5_shape[2], -1 ]) self.softmax_cam = pixelSoftmax(self.reshape_cam) self.vis_cam = tf.transpose(self.softmax_cam, [0, 3, 1, 2]) self.classRank = tf.reduce_mean(self.vis_cam, reduction_indices=[2, 3]) with tf.name_scope("Loss"): #Define loss #self.loss = tf.reduce_mean(-tf.reduce_sum(self.gt * tf.log(self.est+self.epsilon), reduction_indices=[1])) self.loss = tf.reduce_mean(-tf.reduce_sum( self.norm_gt * tf.log(self.softmax_cam + self.epsilon), reduction_indices=[3])) self.regLoss = self.loss + self.regStrength * tf.add_n( [tf.nn.l2_loss(v) for v in tf.trainable_variables()]) self.nan_check_loss = tf.verify_tensor_all_finite( self.loss, msg="check_nan") with tf.name_scope("Opt"): #Define optimizer self.optimizerAll = tf.train.AdamOptimizer( self.learningRate, beta1=self.beta1, beta2=self.beta2, epsilon=self.epsilon).minimize(self.loss) #self.optimizerAll = tf.train.MomentumOptimizer(self.learningRate, momentum=self.beta1).minimize(self.loss) self.optimizerPre = tf.train.AdamOptimizer( self.learningRate, beta1=self.beta1, beta2=self.beta2, epsilon=self.epsilon).minimize(self.loss, var_list=[ self.W_gap, self.B_gap, ]) with tf.name_scope("Metric"): self.correct = tf.equal( tf.argmax( tf.reduce_mean(self.gt, reduction_indices=[1, 2]), 1), tf.argmax(self.est, 1)) self.accuracy = tf.reduce_mean( tf.cast(self.correct, tf.float32)) #Cannot be on GPU #Remove distractor class rankNoDist = self.classRank[:, :self.numClasses - 1] (self.eval_vals, self.eval_idx) = tf.nn.top_k(rankNoDist, k=5) #Summaries tf.scalar_summary('loss', self.loss, name="lossSum") tf.scalar_summary('accuracy', self.accuracy, name="accSum") tf.histogram_summary('input', self.inputImage, name="image_vis") tf.histogram_summary('gt', self.gt, name="gt_vis") tf.histogram_summary('norm_gt', self.norm_gt, name="norm_gt_vis") #Conv layer histograms tf.histogram_summary('conv1_1', self.h_conv1_1, name="conv1_1_vis") tf.histogram_summary('conv1_2', self.h_conv1_2, name="conv1_2_vis") tf.histogram_summary('conv2_1', self.h_conv2_1, name="conv2_1_vis") tf.histogram_summary('conv2_2', self.h_conv2_2, name="conv2_2_vis") tf.histogram_summary('conv3_1', self.h_conv3_1, name="conv3_1_vis") tf.histogram_summary('conv3_2', self.h_conv3_2, name="conv3_2_vis") tf.histogram_summary('conv3_3', self.h_conv3_3, name="conv3_3_vis") tf.histogram_summary('conv4_1', self.h_conv4_1, name="conv4_1_vis") tf.histogram_summary('conv4_2', self.h_conv4_2, name="conv4_2_vis") tf.histogram_summary('conv4_3', self.h_conv4_3, name="conv4_3_vis") tf.histogram_summary('conv5_1', self.h_conv5_1, name="conv5_1_vis") tf.histogram_summary('conv5_2', self.h_conv5_2, name="conv5_2_vis") tf.histogram_summary('conv5_3', self.h_conv5_3, name="conv5_3_vis") tf.histogram_summary('cam', self.flat_cam, name="flat_cam") tf.histogram_summary('softmax_cam', self.softmax_cam, name="softmax_cam") tf.histogram_summary('gap', self.h_gap, name="gap_vis") tf.histogram_summary('est', self.est, name="est_vis") #Weight and bias hists tf.histogram_summary('w_conv1_1', self.W_conv1_1, name="w_conv1_1_vis") tf.histogram_summary('b_conv1_1', self.B_conv1_1, name="b_conv1_1_vis") tf.histogram_summary('w_conv1_2', self.W_conv1_2, name="w_conv1_2_vis") tf.histogram_summary('b_conv1_2', self.B_conv1_2, name="b_conv1_2_vis") tf.histogram_summary('w_conv2_1', self.W_conv2_1, name="w_conv2_1_vis") tf.histogram_summary('b_conv2_1', self.B_conv2_1, name="b_conv2_1_vis") tf.histogram_summary('w_conv2_2', self.W_conv2_2, name="w_conv2_2_vis") tf.histogram_summary('b_conv2_2', self.B_conv2_2, name="b_conv2_2_vis") tf.histogram_summary('w_conv3_1', self.W_conv3_1, name="w_conv3_1_vis") tf.histogram_summary('b_conv3_1', self.B_conv3_1, name="b_conv3_1_vis") tf.histogram_summary('w_conv3_2', self.W_conv3_2, name="w_conv3_2_vis") tf.histogram_summary('b_conv3_2', self.B_conv3_2, name="b_conv3_2_vis") tf.histogram_summary('w_conv3_3', self.W_conv3_3, name="w_conv3_3_vis") tf.histogram_summary('b_conv3_3', self.B_conv3_3, name="b_conv3_3_vis") tf.histogram_summary('w_conv4_1', self.W_conv4_1, name="w_conv4_1_vis") tf.histogram_summary('b_conv4_1', self.B_conv4_1, name="b_conv4_1_vis") tf.histogram_summary('w_conv4_2', self.W_conv4_2, name="w_conv4_2_vis") tf.histogram_summary('b_conv4_2', self.B_conv4_2, name="b_conv4_2_vis") tf.histogram_summary('w_conv4_3', self.W_conv4_3, name="w_conv4_3_vis") tf.histogram_summary('b_conv4_3', self.B_conv4_3, name="b_conv4_3_vis") tf.histogram_summary('w_conv5_1', self.W_conv5_1, name="w_conv5_1_vis") tf.histogram_summary('b_conv5_1', self.B_conv5_1, name="b_conv5_1_vis") tf.histogram_summary('w_conv5_2', self.W_conv5_2, name="w_conv5_2_vis") tf.histogram_summary('b_conv5_2', self.B_conv5_2, name="b_conv5_2_vis") tf.histogram_summary('w_conv5_3', self.W_conv5_3, name="w_conv5_3_vis") tf.histogram_summary('b_conv5_3', self.B_conv5_3, name="b_conv5_3_vis") tf.histogram_summary('w_gap', self.W_gap, name="w_gap_vis") tf.histogram_summary('b_gap', self.B_gap, name="b_gap_vis")
def buildModel(self, inMatFilename = None): if(inMatFilename): npWeights = loadWeights(inMatFilename) #Put all conv layers on gpu with tf.device('gpu:0'): with tf.name_scope("inputOps"): inputShape = self.dataObj.inputShape #Get convolution variables as placeholders self.inputImage = node_variable([None, inputShape[0], inputShape[1], inputShape[2]], "inputImage") self.gt = node_variable([None, 1], "gt") #Model variables for convolutions with tf.name_scope("Conv1Ops"): if(inMatFilename): self.W_conv1 = weight_variable_fromnp(npWeights["conv1_w"], "w_conv1") self.B_conv1 = weight_variable_fromnp(npWeights["conv1_b"], "b_conv1") else: self.W_conv1 = weight_variable_fromnp(np.zeros((11, 11, 3, 64), dtype=np.float32), "w_conv1") self.B_conv1 = weight_variable_fromnp(np.zeros((64), dtype=np.float32), "b_conv1") ##First conv layer is 11x11, 3 input channels into 64 output channels #self.W_conv1 = weight_variable_xavier([11, 11, 3, 64], "w_conv1", conv=True) #self.B_conv1 = bias_variable([64], "b_conv1") self.h_conv1 = tf.nn.relu(conv2d(self.inputImage, self.W_conv1, "conv1") + self.B_conv1) self.h_norm1 = tf.nn.local_response_normalization(self.h_conv1, name="LRN1") #relu is communative op, so do relu after pool for efficiency self.h_pool1 = maxpool_2x2(self.h_norm1, "pool1") with tf.name_scope("Conv2Ops"): #Second conv layer is 5x5 conv, into 256 output channels if(inMatFilename): self.W_conv2 = weight_variable_fromnp(npWeights["conv2_w"], "w_conv2") self.B_conv2 = weight_variable_fromnp(npWeights["conv2_b"], "b_conv2") else: self.W_conv2 = weight_variable_fromnp(np.zeros((5, 5, 64, 256), dtype=np.float32), "w_conv2") self.B_conv2 = weight_variable_fromnp(np.zeros((256), dtype=np.float32), "b_conv2") #self.W_conv2 = weight_variable_xavier([5, 5, 64, 256], "w_conv2", conv=True) #self.B_conv2 = bias_variable([256], "b_conv2") self.h_conv2 = tf.nn.relu(conv2d(self.h_pool1, self.W_conv2, "conv2") + self.B_conv2) self.h_norm2 = tf.nn.local_response_normalization(self.h_conv2, name="LRN2") self.h_pool2 = maxpool_2x2(self.h_norm2, "pool2") #Third layer is 3x3 conv into 256 output channels #No pooling with tf.name_scope("Conv3Ops"): #Second conv layer is 5x5 conv, into 256 output channels if(inMatFilename): self.W_conv3 = weight_variable_fromnp(npWeights["conv3_w"], "w_conv3") self.B_conv3 = weight_variable_fromnp(npWeights["conv3_b"], "b_conv3") else: self.W_conv3 = weight_variable_fromnp(np.zeros((3, 3, 256, 256), dtype=np.float32), "w_conv3") self.B_conv3 = weight_variable_fromnp(np.zeros((256), dtype=np.float32), "b_conv3") #self.W_conv3 = weight_variable_xavier([3, 3, 256, 256], "w_conv3", conv=True) #self.B_conv3 = bias_variable([256], "b_conv3") self.h_conv3 = tf.nn.relu(conv2d(self.h_pool2, self.W_conv3, "conv3") + self.B_conv3, name="relu3") #Fourth layer is 3x3 conv into 256 output channels #No pooling with tf.name_scope("Conv4Ops"): #Second conv layer is 5x5 conv, into 256 output channels if(inMatFilename): self.W_conv4 = weight_variable_fromnp(npWeights["conv4_w"], "w_conv4") self.B_conv4 = weight_variable_fromnp(npWeights["conv4_b"], "b_conv4") else: self.W_conv4 = weight_variable_fromnp(np.zeros((3, 3, 256, 256), dtype=np.float32), "w_conv4") self.B_conv4 = weight_variable_fromnp(np.zeros((256), dtype=np.float32), "b_conv4") #self.W_conv4 = weight_variable_xavier([3, 3, 256, 256], "w_conv4", conv=True) #self.B_conv4 = bias_variable([256], "b_conv4") self.h_conv4 = tf.nn.relu(conv2d(self.h_conv3, self.W_conv4, "conv4") + self.B_conv4, name="relu4") #Fifth layer is 3x3 conv into 256 output channels #with pooling with tf.name_scope("Conv5Ops"): #Second conv layer is 5x5 conv, into 256 output channels if(inMatFilename): self.W_conv5 = weight_variable_fromnp(npWeights["conv5_w"], "w_conv5") self.B_conv5 = weight_variable_fromnp(npWeights["conv5_b"], "b_conv5") else: self.W_conv5 = weight_variable_fromnp(np.zeros((3, 3, 256, 256), dtype=np.float32), "w_conv5") self.B_conv5 = weight_variable_fromnp(np.zeros((256), dtype = np.float32), "b_conv5") #self.W_conv5 = weight_variable_xavier([3, 3, 256, 256], "w_conv5", conv=True) #self.B_conv5 = bias_variable([256], "b_conv5") self.h_conv5 = tf.nn.relu(conv2d(self.h_conv4, self.W_conv5, "conv5") + self.B_conv5) self.h_norm5 = tf.nn.local_response_normalization(self.h_conv5, name="LRN5") self.h_pool5 = maxpool_2x2(self.h_norm5, "pool5") #6th layer (not in paper) is 3x3 conv into 256 output channels #with pooling with tf.name_scope("Conv6Ops"): self.W_conv6 = weight_variable_xavier([3, 3, 256, 256], "w_conv6", conv=True) self.B_conv6 = bias_variable([256], "b_conv6") self.h_conv6 = conv2d(self.h_pool5, self.W_conv6, "conv6") + self.B_conv6 self.h_pool6 = tf.nn.relu(maxpool_2x2(self.h_conv6, "pool6"), name="relu6") self.keep_prob = tf.placeholder(tf.float32) #Next is 3 fully connected layers #We should have downsampled by 8 at this point #fc1 should have 4096 channels numInputs = (inputShape[0]/16) * (inputShape[1]/16) * 256 with tf.name_scope("FC1"): self.W_fc1 = weight_variable([numInputs, 2048], "w_fc1", 1e-6) self.B_fc1 = bias_variable([2048], "b_fc1") h_pool6_flat = tf.reshape(self.h_pool6, [-1, numInputs], name="pool6_flat") self.h_fc1 = tf.nn.relu(tf.matmul(h_pool6_flat, self.W_fc1, name="fc1") + self.B_fc1, "fc1_relu") self.h_fc1_drop = tf.nn.dropout(self.h_fc1, self.keep_prob) #Put all opt layers on cpu with tf.device('/cpu:0'): #fc2 should have 128 channels with tf.name_scope("FC2"): self.W_fc2 = weight_variable_xavier([2048, 128], "w_fc2", conv=False) self.B_fc2 = bias_variable([128], "b_fc2") self.h_fc2 = tf.nn.relu(tf.matmul(self.h_fc1_drop, self.W_fc2, name="fc2") + self.B_fc2, "fc2_relu") self.h_fc2_drop = tf.nn.dropout(self.h_fc2, self.keep_prob) #fc3 should have 16 channels #fc3 also uses a sigmoid function #We change it to tanh with tf.name_scope("FC3"): self.W_fc3 = weight_variable_xavier([128, 16], "w_fc3", conv=False) self.B_fc3 = bias_variable([16], "b_fc3") self.h_fc3 = tf.tanh(tf.matmul(self.h_fc2, self.W_fc3, name="fc3") + self.B_fc3, "fc3_relu") #Finally, fc4 condenses into 1 output value with tf.name_scope("FC4"): self.W_fc4 = weight_variable_xavier([16, 1], "w_fc4", conv=False) self.B_fc4 = bias_variable([1], "b_fc4") self.est = tf.matmul(self.h_fc3, self.W_fc4, name="est") + self.B_fc4 with tf.name_scope("Loss"): #Define loss self.loss = tf.reduce_mean(tf.square(self.gt - self.est))/2 with tf.name_scope("Opt"): #Define optimizer #self.optimizerAll = tf.train.AdagradOptimizer(self.learningRate).minimize(self.loss) #self.optimizerFC = tf.train.AdagradOptimizer(self.learningRate).minimize(self.loss, self.optimizerAll = tf.train.AdamOptimizer(self.learningRate).minimize(self.loss) self.optimizerFC = tf.train.AdamOptimizer(self.learningRate).minimize(self.loss, var_list=[self.W_conv6, self.B_conv6, self.W_fc1, self.B_fc1, self.W_fc2, self.B_fc2, self.W_fc3, self.B_fc3, self.W_fc4, self.B_fc4] ) #Summaries tf.scalar_summary('l2 loss', self.loss) tf.histogram_summary('input', self.inputImage) tf.histogram_summary('gt', self.gt) tf.histogram_summary('conv1', self.h_pool1) tf.histogram_summary('conv2', self.h_pool2) tf.histogram_summary('conv3', self.h_conv3) tf.histogram_summary('conv4', self.h_conv4) tf.histogram_summary('conv5', self.h_pool5) tf.histogram_summary('conv6', self.h_pool6) tf.histogram_summary('fc1', self.h_fc1) tf.histogram_summary('fc2', self.h_fc2) tf.histogram_summary('fc3', self.h_fc3) tf.histogram_summary('est', self.est) tf.histogram_summary('w_conv1', self.W_conv1) tf.histogram_summary('b_conv1', self.B_conv1) tf.histogram_summary('w_conv2', self.W_conv2) tf.histogram_summary('b_conv2', self.B_conv2) tf.histogram_summary('w_conv3', self.W_conv3) tf.histogram_summary('b_conv3', self.B_conv3) tf.histogram_summary('w_conv4', self.W_conv4) tf.histogram_summary('b_conv4', self.B_conv4) tf.histogram_summary('w_conv5', self.W_conv5) tf.histogram_summary('b_conv5', self.B_conv5) tf.histogram_summary('w_conv6', self.W_conv6) tf.histogram_summary('b_conv6', self.B_conv6) tf.histogram_summary('w_fc1', self.W_fc1) tf.histogram_summary('b_fc1', self.B_fc1) tf.histogram_summary('w_fc2', self.W_fc2) tf.histogram_summary('b_fc2', self.B_fc2) tf.histogram_summary('w_fc3', self.W_fc3) tf.histogram_summary('b_fc3', self.B_fc3) tf.histogram_summary('w_fc4', self.W_fc4) tf.histogram_summary('b_fc4', self.B_fc4) #Define saver self.saver = tf.train.Saver()
def buildModel(self, inputShape, inMatFilename): if (inMatFilename): npWeights = loadWeights(inMatFilename) #Running on GPU with tf.device('gpu:0'): with tf.name_scope("inputOps"): #Get convolution variables as placeholders self.inputImage = node_variable( [None, inputShape[0], inputShape[1], inputShape[2]], "inputImage") self.gt = node_variable([None, 2], "gt") #Model variables for convolutions with tf.name_scope("Conv1Ops"): if (inMatFilename): self.W_conv1 = weight_variable_fromnp( npWeights["conv1_w"], "w_conv1") self.B_conv1 = weight_variable_fromnp( npWeights["conv1_b"], "b_conv1") else: self.W_conv1 = weight_variable_fromnp( np.zeros((11, 11, 3, 64), dtype=np.float32), "w_conv1") self.B_conv1 = weight_variable_fromnp( np.zeros((64), dtype=np.float32), "b_conv1") #self.W_conv1 = weight_variable_xavier([11, 11, 3, 64], "w_conv1", conv=True) #self.B_conv1 = bias_variable([64], "b_conv1") self.h_conv1 = tf.nn.relu( conv2d(self.inputImage, self.W_conv1, "conv1", stride=[1, 4, 4, 1]) + self.B_conv1) self.h_norm1 = tf.nn.local_response_normalization(self.h_conv1, name="LRN1") self.h_pool1 = maxpool_2x2(self.h_norm1, "pool1") with tf.name_scope("Conv2Ops"): if (inMatFilename): self.W_conv2 = weight_variable_fromnp( npWeights["conv2_w"], "w_conv2") self.B_conv2 = weight_variable_fromnp( npWeights["conv2_b"], "b_conv2") else: self.W_conv2 = weight_variable_fromnp( np.zeros((5, 5, 64, 256), dtype=np.float32), "w_conv2") self.B_conv2 = weight_variable_fromnp( np.zeros((256), dtype=np.float32), "b_conv2") #self.W_conv2 = weight_variable_xavier([5, 5, 64, 256], "w_conv2", conv=True) #self.B_conv2 = bias_variable([256], "b_conv2") self.h_conv2 = tf.nn.relu( conv2d(self.h_pool1, self.W_conv2, "conv2") + self.B_conv2) self.h_norm2 = tf.nn.local_response_normalization(self.h_conv2, name="LRN2") self.h_pool2 = maxpool_2x2(self.h_norm2, "pool2") with tf.name_scope("Conv3Ops"): if (inMatFilename): self.W_conv3 = weight_variable_fromnp( npWeights["conv3_w"], "w_conv3") self.B_conv3 = weight_variable_fromnp( npWeights["conv3_b"], "b_conv3") else: self.W_conv3 = weight_variable_fromnp( np.zeros((3, 3, 256, 256), dtype=np.float32), "w_conv3") self.B_conv3 = weight_variable_fromnp( np.zeros((256), dtype=np.float32), "b_conv3") #self.W_conv3 = weight_variable_xavier([3, 3, 256, 256], "w_conv3", conv=True) #self.B_conv3 = bias_variable([256], "b_conv3") self.h_conv3 = tf.nn.relu( conv2d(self.h_pool2, self.W_conv3, "conv3") + self.B_conv3, name="relu3") with tf.name_scope("Conv4Ops"): if (inMatFilename): self.W_conv4 = weight_variable_fromnp( npWeights["conv4_w"], "w_conv4") self.B_conv4 = weight_variable_fromnp( npWeights["conv4_b"], "b_conv4") else: self.W_conv4 = weight_variable_fromnp( np.zeros((3, 3, 256, 256), dtype=np.float32), "w_conv4") self.B_conv4 = weight_variable_fromnp( np.zeros((256), dtype=np.float32), "b_conv4") #self.W_conv4 = weight_variable_xavier([3, 3, 256, 256], "w_conv4", conv=True) #self.B_conv4 = bias_variable([256], "b_conv4") self.h_conv4 = tf.nn.relu( conv2d(self.h_conv3, self.W_conv4, "conv4") + self.B_conv4, name="relu4") with tf.name_scope("Conv5Ops"): if (inMatFilename): self.W_conv5 = weight_variable_fromnp( npWeights["conv5_w"], "w_conv5") self.B_conv5 = weight_variable_fromnp( npWeights["conv5_b"], "b_conv5") else: self.W_conv5 = weight_variable_fromnp( np.zeros((3, 3, 256, 256), dtype=np.float32), "w_conv5") self.B_conv5 = weight_variable_fromnp( np.zeros((256), dtype=np.float32), "b_conv5") #self.W_conv5 = weight_variable_xavier([3, 3, 256, 256], "w_conv5", conv=True) #self.B_conv5 = bias_variable([256], "b_conv5") self.h_conv5 = tf.nn.relu( conv2d(self.h_conv4, self.W_conv5, "conv5") + self.B_conv5) self.h_pool5 = maxpool_2x2(self.h_conv5, "pool5") #placeholder for specifying dropout self.keep_prob = tf.placeholder(tf.float32) #32 comes from 4 stride in conv1, 2 stride in pool1, 2 stride in pool2, 2 stride in pool5 numInputs = (inputShape[0] / 32) * (inputShape[1] / 32) * 256 with tf.name_scope("FC1"): #if(inMatFilename): # self.W_conv5 = weight_variable_fromnp(npWeights["fc1_w"], "w_fc1") # self.B_conv5 = weight_variable_fromnp(npWeights["fc1_b"], "b_fc1") #else: # self.W_conv5 = weight_variable_fromnp(np.zeros((6*6*256, 4096), dtype=np.float32), "w_fc1") # self.B_conv5 = weight_variable_fromnp(np.zeros((4096), dtype = np.float32), "b_fc1") self.W_fc1 = weight_variable_xavier([numInputs, 4096], "w_fc1") self.B_fc1 = bias_variable([4096], "b_fc1") h_pool5_flat = tf.reshape(self.h_pool5, [-1, numInputs], name="pool5_flat") self.h_fc1 = tf.nn.relu( tf.matmul(h_pool5_flat, self.W_fc1, name="fc1") + self.B_fc1, "fc1_relu") self.h_fc1_drop = tf.nn.dropout(self.h_fc1, self.keep_prob) with tf.name_scope("FC2"): #if(inMatFilename): # self.W_conv5 = weight_variable_fromnp(npWeights["fc2_w"], "w_fc2") # self.B_conv5 = weight_variable_fromnp(npWeights["fc2_b"], "b_fc2") #else: # self.W_conv5 = weight_variable_fromnp(np.zeros((4096, 4096), dtype=np.float32), "w_fc2") # self.B_conv5 = weight_variable_fromnp(np.zeros((4096), dtype = np.float32), "b_fc2") self.W_fc2 = weight_variable_xavier([4096, 4096], "w_fc2") self.B_fc2 = bias_variable([4096], "b_fc2") self.h_fc2 = tf.nn.relu( tf.matmul(self.h_fc1_drop, self.W_fc2, name="fc2") + self.B_fc2, "fc2_relu") self.h_fc2_drop = tf.nn.dropout(self.h_fc2, self.keep_prob) #fc3 should have 16 channels #fc3 also uses a sigmoid function #We change it to tanh with tf.name_scope("FC3"): #if(inMatFilename): # self.W_conv5 = weight_variable_fromnp(npWeights["fc3_w"], "w_fc3") # self.B_conv5 = weight_variable_fromnp(npWeights["fc3_b"], "b_fc3") #else: # self.W_conv5 = weight_variable_fromnp(np.zeros((4096, 2), dtype=np.float32), "w_fc3") # self.B_conv5 = weight_variable_fromnp(np.zeros((2), dtype = np.float32), "b_fc3") self.W_fc3 = weight_variable_xavier([4096, 2], "w_fc3") self.B_fc3 = bias_variable([2], "b_fc3") self.est = tf.nn.softmax( tf.matmul(self.h_fc2_drop, self.W_fc3, name="fc3") + self.B_fc3, "fc3_softmax") with tf.name_scope("Loss"): #Define loss #self.loss = tf.reduce_mean(-tf.reduce_sum(self.gt * tf.log(self.est), reduction_indices=[1])) self.loss = tf.reduce_mean( -(self.gt[:, 1] * .8 * tf.log(self.est[:, 1]) + self.gt[:, 0] * .2 * tf.log(self.est[:, 0]))) with tf.name_scope("Opt"): #Define optimizer #self.optimizerAll = tf.train.AdagradOptimizer(self.learningRate).minimize(self.loss) #self.optimizerFC = tf.train.AdagradOptimizer(self.learningRate).minimize(self.loss, self.optimizerAll = tf.train.AdamOptimizer( self.learningRate).minimize(self.loss) self.optimizerFC = tf.train.AdamOptimizer( self.learningRate).minimize(self.loss, var_list=[ self.W_fc1, self.B_fc1, self.W_fc2, self.B_fc2, self.W_fc3, self.B_fc3 ]) with tf.name_scope("Metric"): self.gtIdx = tf.argmax(self.gt, 1) self.estIdx = tf.argmax(self.est, 1) boolGtIdx = tf.cast(self.gtIdx, tf.bool) boolEstIdx = tf.cast(self.estIdx, tf.bool) #Logical and for true positive lAnd = tf.logical_and(boolGtIdx, boolEstIdx) self.tp = tf.reduce_sum(tf.cast(lAnd, tf.float32)) #Logical nor for true negatives lNor = tf.logical_not(tf.logical_or(boolGtIdx, boolEstIdx)) self.tn = tf.reduce_sum(tf.cast(lNor, tf.float32)) #Subtraction and comparison for others lSub = self.gtIdx - self.estIdx Ones = tf.cast(tf.ones(tf.shape(lSub)), tf.int64) self.fn = tf.reduce_sum( tf.cast(tf.equal(lSub, Ones), tf.float32)) self.fp = tf.reduce_sum( tf.cast(tf.equal(lSub, -Ones), tf.float32)) #Accuracy, precision, and recall calculations self.accuracy = (self.tp + self.tn) / (self.tp + self.tn + self.fp + self.fn) self.precision = self.tp / (self.tp + self.fp) self.recall = self.tp / (self.tp + self.fn) #Summaries tf.scalar_summary('loss', self.loss, name="lossSum") tf.scalar_summary('accuracy', self.accuracy, name="accSum") tf.scalar_summary('precision', self.precision, name="precSum") tf.scalar_summary('recall', self.recall, name="recallSum") tf.scalar_summary('tp', self.tp, name="tp") tf.scalar_summary('fp', self.fp, name="fp") tf.scalar_summary('tn', self.tn, name="tn") tf.scalar_summary('fn', self.fn, name="fn") tf.histogram_summary('input', self.inputImage, name="image") tf.histogram_summary('gt', self.gt, name="gt") tf.histogram_summary('conv1', self.h_pool1, name="conv1") tf.histogram_summary('conv2', self.h_pool2, name="conv2") tf.histogram_summary('conv3', self.h_conv3, name="conv3") tf.histogram_summary('conv4', self.h_conv4, name="conv4") tf.histogram_summary('conv5', self.h_pool5, name="conv5") tf.histogram_summary('fc1', self.h_fc1, name="fc1") tf.histogram_summary('fc2', self.h_fc2, name="fc2") tf.histogram_summary('est', self.est, name="fc3") tf.histogram_summary('w_conv1', self.W_conv1, name="w_conv1") tf.histogram_summary('b_conv1', self.B_conv1, name="b_conv1") tf.histogram_summary('w_conv2', self.W_conv2, name="w_conv2") tf.histogram_summary('b_conv2', self.B_conv2, name="b_conv2") tf.histogram_summary('w_conv3', self.W_conv3, name="w_conv3") tf.histogram_summary('b_conv3', self.B_conv3, name="b_conv3") tf.histogram_summary('w_conv4', self.W_conv4, name="w_conv4") tf.histogram_summary('b_conv4', self.B_conv4, name="b_conv4") tf.histogram_summary('w_conv5', self.W_conv5, name="w_conv5") tf.histogram_summary('b_conv5', self.B_conv5, name="b_conv5") tf.histogram_summary('w_fc1', self.W_fc1, name="w_fc1") tf.histogram_summary('b_fc1', self.B_fc1, name="b_fc1") tf.histogram_summary('w_fc2', self.W_fc2, name="w_fc2") tf.histogram_summary('b_fc2', self.B_fc2, name="b_fc2") tf.histogram_summary('w_fc3', self.W_fc3, name="w_fc3") tf.histogram_summary('b_fc3', self.B_fc3, name="b_fc3") #Define saver self.saver = tf.train.Saver()