def build_compute_loss(self, att_logits, labels): loss_vec=tf.nn.sparse_softmax_cross_entropy_with_logits(logits=att_logits, labels=labels, name=None) loss_cls = tf.reduce_mean(loss_vec) reg_var_list = self.get_variables_by_name([""], False) loss_reg = loss_func.l2_regularization_loss(reg_var_list[""], self.weight_decay) loss = loss_cls + loss_reg return loss, loss_vec
def train_op(self): # Collect variables for training tvars = [ var for var in tf.trainable_variables() if var.op.name.startswith('refer_seg') ] print('Collecting variables for training:') for var in tvars: print('\t%s' % var.name) print('Done.') # Collect variables for regularization rvars = [ var for var in tf.trainable_variables() if var.op.name.startswith('refer_seg') ] print('Collecting variables for regularization:') for var in rvars: print('\t%s' % var.name) print('Done.') # Define loss self.target_coarse = tf.image.resize_bilinear(self.target_fine, [self.vf_h, self.vf_w]) self.cls_loss = loss.weighed_logistic_loss(self.pred, self.target_fine, 1, 1) self.reg_loss = loss.l2_regularization_loss(rvars, self.weight_decay) self.sum_loss = self.cls_loss + self.reg_loss # Define learning rate self.global_step = tf.Variable(0, trainable=False) self.learning_rate = tf.train.polynomial_decay(self.start_lr, self.global_step, self.lr_decay_step, self.end_lr, self.lr_decay_rate) # Define optimization process optimizer = tf.train.AdamOptimizer(self.learning_rate) grads_and_vars = optimizer.compute_gradients(self.sum_loss, var_list=tvars) var_lr_mult = {} for var in tvars: var_lr_mult[var] = 2.0 if var.op.name.find('biases') > 0 else 1.0 print('Setting variable learning rate multiplication:') for var in tvars: print('\t%s: %f' % (var.name, var_lr_mult[var])) print('Done.') grads_and_vars = [ ((g if var_lr_mult[v] == 1 else tf.multiply(var_lr_mult[v], g)), v) for g, v in grads_and_vars ] self.train_step = optimizer.apply_gradients( grads_and_vars, global_step=self.global_step)
def train_op(self): if self.conv5: tvars = [var for var in tf.trainable_variables() if var.op.name.startswith('text_objseg') or var.name.startswith('res5') or var.name.startswith('res4') or var.name.startswith('res3')] else: tvars = [var for var in tf.trainable_variables() if var.op.name.startswith('text_objseg')] reg_var_list = [var for var in tvars if var.op.name.find(r'DW') > 0 or var.name[-9:-2] == 'weights'] print('Collecting variables for regularization:') for var in reg_var_list: print('\t%s' % var.name) print('Done.') # define loss self.target = tf.image.resize_bilinear(self.target_fine, [self.vf_h, self.vf_w]) self.cls_loss_c5 = loss.weighed_logistic_loss(self.up_c5, self.target_fine, 1, 1) self.cls_loss_c4 = loss.weighed_logistic_loss(self.up_c4, self.target_fine, 1, 1) self.cls_loss_c3 = loss.weighed_logistic_loss(self.up_c3, self.target_fine, 1, 1) self.cls_loss = loss.weighed_logistic_loss(self.up, self.target_fine, 1, 1) self.cls_loss_all = 0.7 * self.cls_loss + 0.1 * self.cls_loss_c5 \ + 0.1 * self.cls_loss_c4 + 0.1 * self.cls_loss_c3 self.reg_loss = loss.l2_regularization_loss(reg_var_list, self.weight_decay) self.cost = self.cls_loss_all + self.reg_loss # learning rate lr = tf.Variable(0.0, trainable=False) self.learning_rate = tf.train.polynomial_decay(self.start_lr, lr, self.lr_decay_step, end_learning_rate=0.00001, power=0.9) # optimizer if self.optimizer == 'adam': optimizer = tf.train.AdamOptimizer(self.learning_rate) else: raise ValueError("Unknown optimizer type %s!" % self.optimizer) # learning rate multiplier grads_and_vars = optimizer.compute_gradients(self.cost, var_list=tvars) var_lr_mult = {} for var in tvars: if var.op.name.find(r'biases') > 0: var_lr_mult[var] = 2.0 elif var.name.startswith('res5') or var.name.startswith('res4') or var.name.startswith('res3'): var_lr_mult[var] = 1.0 else: var_lr_mult[var] = 1.0 print('Variable learning rate multiplication:') for var in tvars: print('\t%s: %f' % (var.name, var_lr_mult[var])) print('Done.') grads_and_vars = [((g if var_lr_mult[v] == 1 else tf.multiply(var_lr_mult[v], g)), v) for g, v in grads_and_vars] # training step self.train_step = optimizer.apply_gradients(grads_and_vars, global_step=lr)
def train_op(self): # define loss self.target = tf.image.resize_bilinear(self.target_fine, [self.vf_h, self.vf_w]) tvars = [ var for var in tf.trainable_variables() if var.op.name.startswith('text_objseg') or var.op.name.startswith('ResNet/fc1000') ] reg_var_list = [var for var in tvars if var.op.name.find(r'DW') > 0] self.mid_loss_5 = loss.weighed_logistic_loss(self.up1_5, self.target) self.mid_loss_4 = loss.weighed_logistic_loss(self.up1_4, self.target) self.mid_loss_3 = loss.weighed_logistic_loss(self.up1_3, self.target) self.cls_loss = loss.weighed_logistic_loss(self.pred, self.target) self.reg_loss = loss.l2_regularization_loss(reg_var_list, self.weight_decay) self.cost = self.cls_loss + self.reg_loss + self.mid_loss_5 + self.mid_loss_4 + self.mid_loss_3 # learning rate lr = tf.Variable(0.0, trainable=False) self.learning_rate = tf.train.polynomial_decay( self.start_lr, lr, self.lr_decay_step, end_learning_rate=0.00001, power=0.9) # optimizer if self.optimizer == 'adam': optimizer = tf.train.AdamOptimizer(self.learning_rate) else: raise ValueError("Unknown optimizer type %s!" % self.optimizer) # learning rate multiplier grads_and_vars = optimizer.compute_gradients(self.cost, var_list=tvars) var_lr_mult = { var: (2.0 if var.op.name.find(r'biases') > 0 else 1.0) for var in tvars } grads_and_vars = [ ((g if var_lr_mult[v] == 1 else tf.multiply(var_lr_mult[v], g)), v) for g, v in grads_and_vars ] # training step self.train_step = optimizer.apply_gradients(grads_and_vars, global_step=lr)
def build_model(self): self.imcrop_batch = tf.placeholder( tf.float32, [self.batch_size, self.input_H, self.input_W, 3]) self.text_seq_batch = tf.placeholder(tf.int32, [self.rnn_cells, self.batch_size]) self.label_batch = tf.placeholder( tf.float32, [self.batch_size, self.input_H, self.input_W, 1]) self.scores = self.forward(self.imcrop_batch, self.text_seq_batch) self.train_var_list = self.get_train_var_list() #Calculate loss self.cls_loss = loss.weighed_logistic_loss(self.scores, self.label_batch) #Add regularization to weight matrices (excluding bias) reg_var_list = [ var for var in tf.trainable_variables() if (var in self.train_var_list) and ( var.name[-9:-2] == 'weights' or var.name[-8:-2] == 'Matrix') ] reg_loss = loss.l2_regularization_loss(reg_var_list, self.weight_decay) self.total_loss = self.cls_loss + reg_loss
var_lr_mult = { var: (deeplab_lr_mult if var.name.startswith('deeplab') else 1.0) for var in train_var_list } print('Variable learning rate multiplication:') for var in train_var_list: print('\t%s: %f' % (var.name, var_lr_mult[var])) print('Done.') ################################################################################ # Loss function and accuracy ################################################################################ cls_loss = loss.weighed_logistic_loss(scores, label_batch, pos_loss_mult, neg_loss_mult) reg_loss = loss.l2_regularization_loss(reg_var_list, weight_decay) total_loss = cls_loss + reg_loss ################################################################################ # Solver ################################################################################ global_step = tf.Variable(0, trainable=False) learning_rate = tf.train.exponential_decay(start_lr, global_step, lr_decay_step, lr_decay_rate, staircase=True) solver = tf.train.MomentumOptimizer(learning_rate=learning_rate, momentum=momentum) # Compute gradients
def train_op(self): if self.conv5: tvars = [ var for var in tf.trainable_variables() if var.op.name.startswith('text_objseg') or var.name.startswith('res5') or var.name.startswith('res4') or var.name.startswith('res3') ] else: tvars = [ var for var in tf.trainable_variables() if var.op.name.startswith('text_objseg') ] if self.freeze_bn: tvars = [ var for var in tvars if 'beta' not in var.name and 'gamma' not in var.name ] reg_var_list = [ var for var in tvars if var.op.name.find(r'DW') > 0 or var.name[-9:-2] == 'weights' ] print('Collecting variables for regularization:') for var in reg_var_list: print('\t%s' % var.name) print('Done.') # define loss self.target = tf.image.resize_bilinear(self.target_fine, [self.vf_h, self.vf_w]) self.cls_loss_c5 = loss.weighed_logistic_loss(self.up_c5, self.target_fine, 1, 1) self.cls_loss_c4 = loss.weighed_logistic_loss(self.up_c4, self.target_fine, 1, 1) # self.cls_loss_c3 = loss.weighed_logistic_loss(self.up_c3, self.target_fine, 1, 1) self.cls_loss = loss.weighed_logistic_loss(self.up, self.target_fine, 1, 1) self.cls_loss_all = 0.8 * self.cls_loss + 0.1 * self.cls_loss_c5 \ + 0.1 * self.cls_loss_c4 self.reg_loss = loss.l2_regularization_loss(reg_var_list, self.weight_decay) self.cost = self.cls_loss_all + self.reg_loss # learning rate self.train_step = tf.Variable(0, trainable=False) self.learning_rate = tf.train.polynomial_decay( self.start_lr, self.train_step, self.lr_decay_step, end_learning_rate=0.00001, power=0.9) # optimizer if self.optimizer == 'adam': optimizer = tf.train.AdamOptimizer(self.learning_rate) else: raise ValueError("Unknown optimizer type %s!" % self.optimizer) # learning rate multiplier grads_and_vars = optimizer.compute_gradients(self.cost, var_list=tvars) var_lr_mult = {} for var in tvars: if var.op.name.find(r'biases') > 0: var_lr_mult[var] = 2.0 elif var.name.startswith('res5') or var.name.startswith( 'res4') or var.name.startswith('res3'): var_lr_mult[var] = 1.0 else: var_lr_mult[var] = 1.0 print('Variable learning rate multiplication:') for var in tvars: print('\t%s: %f' % (var.name, var_lr_mult[var])) print('Done.') grads_and_vars = [ ((g if var_lr_mult[v] == 1 else tf.multiply(var_lr_mult[v], g)), v) for g, v in grads_and_vars ] # training step update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): self.train = optimizer.apply_gradients(grads_and_vars, global_step=self.train_step) # Summary in tensorboard tf.summary.scalar('loss_all', self.cls_loss_all) # tf.summary.scalar('loss_c3', self.cls_loss_c3) tf.summary.scalar('loss_c4', self.cls_loss_c4) tf.summary.scalar('loss_c5', self.cls_loss_c5) tf.summary.scalar('loss_last', self.cls_loss) pred = tf.convert_to_tensor(tf.cast(self.up > 0, tf.int32), tf.int32) labl = self.target_fine intersect = tf.reduce_sum(tf.cast( tf.logical_and(tf.cast(pred, tf.bool), tf.cast(labl, tf.bool)), tf.int32), axis=(1, 2, 3)) union = tf.reduce_sum(tf.cast( tf.logical_or(tf.cast(pred, tf.bool), tf.cast(labl, tf.bool)), tf.int32), axis=(1, 2, 3)) self.mIoU = tf.reduce_mean(tf.divide(intersect, union)) tf.summary.scalar('mean_IOU', self.mIoU) self.merged = tf.summary.merge_all()
print('Done.') # Collect learning rate for trainable variables var_lr_mult = {var: (vgg_lr_mult if var.name.startswith('vgg_local') else 1.0) for var in train_var_list} print('Variable learning rate multiplication:') for var in train_var_list: print('\t%s: %f' % (var.name, var_lr_mult[var])) print('Done.') ################################################################################ # Loss function and accuracy ################################################################################ cls_loss = loss.weighed_logistic_loss(scores, label_batch, pos_loss_mult, neg_loss_mult) reg_loss = loss.l2_regularization_loss(reg_var_list, weight_decay) total_loss = cls_loss + reg_loss def compute_accuracy(scores, labels): is_pos = (labels != 0) is_neg = np.logical_not(is_pos) num_all = labels.shape[0] num_pos = np.sum(is_pos) num_neg = num_all - num_pos is_correct = np.logical_xor(scores < 0, is_pos) accuracy_all = np.sum(is_correct) / num_all accuracy_pos = np.sum(is_correct[is_pos]) / num_pos accuracy_neg = np.sum(is_correct[is_neg]) / num_neg return accuracy_all, accuracy_pos, accuracy_neg
def train_op(self): # Collect variables for training tvars = [ var for var in tf.trainable_variables() if var.op.name.startswith('refer_seg') ] print('Collecting variables for training:') for var in tvars: print('\t%s' % var.name) print('Done.') # Collect variables for regularization rvars = [ var for var in tf.trainable_variables() if var.op.name.startswith('refer_seg') ] print('Collecting variables for regularization:') for var in rvars: print('\t%s' % var.name) print('Done.') self.target_coarse = tf.image.resize_bilinear(self.target_fine, [self.vf_h, self.vf_w]) self.mask_indices = tf.slice(tf.where(self.target_coarse > 0.5), [0, 0], [-1, 3]) #self.masked_score = tf.gather(self.score, self.mask_indices) gathered_score = [] for b in range(self.batch_size): mask_indices = tf.where( tf.slice(self.target_coarse, [b, 0, 0, 0], [1, -1, -1, -1]) > 0.5) mask_indices = tf.slice(mask_indices, [0, 0], [-1, 3]) print(mask_indices.get_shape().as_list()) masked_score = tf.gather_nd( tf.slice(self.score, [b, 0, 0, 0], [1, -1, -1, -1]), mask_indices) print(masked_score.get_shape().as_list()) gathered_score.append(masked_score) self.gathered_score = tf.stack(gathered_score) print(self.gathered_score.get_shape().as_list()) #self.masked_score = tf.gather_nd(self.score, self.mask_indices) self.mean_score = tf.reduce_mean(self.gathered_score, axis=1) self.spk_loss = tf.reduce_sum( tf.squared_difference(self.mean_score, self.encoding)) self.reg_loss = loss.l2_regularization_loss(rvars, self.weight_decay) self.sum_loss = self.spk_loss + self.reg_loss # Define learning rate self.global_step = tf.Variable(0, trainable=False) self.learning_rate = tf.train.polynomial_decay(self.start_lr, self.global_step, self.lr_decay_step, self.end_lr, self.lr_decay_rate) # Define optimization process optimizer = tf.train.AdamOptimizer(self.learning_rate) grads_and_vars = optimizer.compute_gradients(self.sum_loss, var_list=tvars) var_lr_mult = {} for var in tvars: var_lr_mult[var] = 2.0 if var.op.name.find('biases') > 0 else 1.0 print('Setting variable learning rate multiplication:') for var in tvars: print('\t%s: %f' % (var.name, var_lr_mult[var])) print('Done.') grads_and_vars = [ ((g if var_lr_mult[v] == 1 else tf.multiply(var_lr_mult[v], g)), v) for g, v in grads_and_vars ] self.train_step = optimizer.apply_gradients( grads_and_vars, global_step=self.global_step)