def getSIFTTestBatch(sift_test_list, batch_id): img_batch = np.zeros([batch_size, image_size, image_size, 3], np.float32) start_id = batch_id * batch_size end_id = (batch_id + 1) * batch_size if (end_id > len(sift_test_list)): print("------ Error reading sift test batch!") return None # read batch data = Data(batch_size, image_size, pcl_size, None, None) list_batch = sift_test_list[start_id:end_id] for i in range(len(list_batch)): img = cv2.imread(list_batch[i]) img = data.img_augmentation(img) img_batch[i, :, :, :] = img return img_batch
def getISSTestBatch(iss_test_list, batch_id): pos_pcl_batch = np.zeros([batch_size, pcl_size, 3], np.float32) start_id = batch_id * batch_size end_id = (batch_id + 1) * batch_size if (end_id > len(iss_test_list)): print("------ Error reading sift test batch!") return None # read batch data = Data(batch_size, image_size, pcl_size, None, None) list_batch = iss_test_list[start_id:end_id] for i in range(len(list_batch)): pos_pcl = data.read_pcd(list_batch[i]) # > 1024 points if pos_pcl.shape[0] > pcl_size: random_id = np.random.permutation(pos_pcl.shape[0]) pos_pcl_batch[i, :, :] = pos_pcl[random_id[0:pcl_size]] else: pos_pcl_batch[i, 0:pos_pcl.shape[0], :] = pos_pcl return pos_pcl_batch
def train(load_version, train_list, test_list): print('----------------- START to train -----------------') new_test_list = test_list[0][1] + test_list[0][2] print new_test_list[1] data = Data(batch_size, image_size, pcl_size, train_list, new_test_list) total_test_num = (len(new_test_list) // batch_size) * batch_size # data = Data(batch_size, image_size, pcl_size, # train_list, test_list[0][1]) # total_test_num = (len(test_list[0][1]) // batch_size) * batch_size # data = Data(batch_size, image_size, pcl_size, # train_list, train_list[0][779:907]) # record test_list for checking # with open('test_list.txt', 'w') as file: # for line in test_list[0][100]: # file.writelines('%s %s %s %s\n' % (line.submap_id,line.cam_id, line.sift_filename,line.iss_filename)) # define placeholder image_pl = tf.placeholder(tf.float32, shape=[batch_size, image_size, image_size, 3]) pos_pcl_pl = tf.placeholder(tf.float32, shape=[batch_size, pcl_size, 3]) neg_pcl_pl = tf.placeholder(tf.float32, shape=[batch_size, pcl_size, 3]) is_training = tf.placeholder(tf.bool) #label_pl = tf.placeholder(tf.int32 , shape=[batch_size, 1]) learning_rate = tf.placeholder(tf.float32) # tensorboard: visualise sift image patch #tf.summary.image('input_sift_image', image_pl, 64) # build model print('build model') with tf.device('/gpu:1'): # use gpu 1 to forward with tf.variable_scope('image_branch') as scope: image_feature = vgg16(image_pl, is_training=True, output_dim=image_feature_dim, bn_decay=None) with tf.variable_scope('pointcloud_branch') as scope: pos_pcl_feature, _ = pointNet(pos_pcl_pl, pcl_feature_dim, is_training=is_training, use_bn=False, bn_decay=None) scope.reuse_variables() neg_pcl_feature, _ = pointNet(neg_pcl_pl, pcl_feature_dim, is_training=is_training, use_bn=False, bn_decay=None) # define loss print('define loss...') loss = triplet_loss(image_feature, pos_pcl_feature, neg_pcl_feature, 0) # tensorboard: visualise loss tf.summary.scalar('loss', loss) # set training print('set training...') with tf.device('/gpu:0'): # use gpu 0 to backward # set global step global_step = tf.Variable(0, trainable=False) # set learning optimisation with tf.name_scope('train'): train_step = tf.train.AdamOptimizer( learning_rate, 0.9, 0.999).minimize(loss, global_step=global_step) saver = tf.train.Saver(tf.all_variables(), max_to_keep=None) # tf.global_variables # run model print('run model...') config = tf.ConfigProto(log_device_placement=False, allow_soft_placement=True) config.gpu_options.allow_growth = True config.gpu_options.per_process_gpu_memory_fraction = 0.9 with tf.Session(config=config) as sess: # summary #print('initialise tensorboard...') # merged = tf.summary.merge_all() save_version = 'v1' # train_writer = tf.summary.FileWriter('tensorboard/' + save_version + '/train', sess.graph) # test_writer = tf.summary.FileWriter('tensorboard/' + save_version + '/test') print('initialise model...') sess.run(tf.global_variables_initializer()) print(' load model...') save_path = 'model/' + 'v1' + '/' + load_version + '_model.ckpt' saver.restore(sess, save_path) #restore_tf_model(sess) print(" Model loaded from: %s" % save_path) # Train and Test global_step_val = 0 for epoch in range(epoch_time): epoch += 31 num_of_iterations = 0 # # --------------------- evaluate model --------------------------- # print('**** Validate ...') # print(' Compute image and pcl descriptors') # # # test the first run /test_list[0] only # # total_test_num = (len(test_list[0][1]) // batch_size) * batch_size # #total_test_num = (len(train_list[0][779:907]) // batch_size) * batch_size # # img_feature = np.zeros([total_test_num, image_feature_dim]) # pcl_feature = np.zeros([total_test_num, pcl_feature_dim]) # # batch_counter = 0 # # # feed test list into network # while True: # # read a batch # img_batch, pcl_batch = data.getTestBatch() # # return None, end of this epoch # if img_batch is None: # break # # # feed batch into network # feed_dict = {image_pl: img_batch, pos_pcl_pl: pcl_batch, is_training: False} # img_batch_feature, pcl_batch_feature = sess.run([image_feature, pos_pcl_feature], feed_dict=feed_dict) # img_feature[batch_counter: batch_counter+img_batch_feature.shape[0],:] = img_batch_feature # pcl_feature[batch_counter: batch_counter+pcl_batch_feature.shape[0],:] = pcl_batch_feature # # batch_counter += img_batch_feature.shape[0] # # print(' Compute top 1 accuracy') # # compute top1 accuracy and record data # val_accuracy = ComputeAccuracy(img_feature, pcl_feature) # with open('tensorboard/' + 'v2_trainlist_accuracy.txt', 'a') as file: # file.write(str(epoch) + ' ' + str(global_step_val) + ' : ' + str(val_accuracy)+'\n') # #file.write(str(epoch) + ' ' + str(global_step_val) + ' : ' + str(val_accuracy)+ ', model version: '+ save_version+'\n') # print(' global step: %d, accuracy = %.3f%%' % (global_step_val, val_accuracy*100.0)) # # ---------------------------------------------------------------- # --------------------- train model ---------------------- # shuffle train list data.train_list = shuffleTrainset(train_list) while True: if num_of_iterations % 2000 == 0: # save model save_version = 'v2_' + str(epoch) + '_' + str( num_of_iterations) save_path = saver.save( sess, 'model/v1' + '/' + save_version + '_model.ckpt') print(" Model saved in file: %s" % save_path) # -------------------- evaluate model --------------------- print('**** Validate ...') print(' Compute image and pcl descriptors') # test the first run /test_lists[0] only #total_test_num = (len(train_list[0][779:907]) // batch_size) * batch_size # total_test_num = (len(test_list[0][1]) // batch_size) * batch_size img_feature = np.zeros([total_test_num, image_feature_dim]) pcl_feature = np.zeros([total_test_num, pcl_feature_dim]) batch_counter = 0 # feed test list into network while True: # read a batch img_batch, pcl_batch = data.getTestBatch() # return None, end of this epoch if img_batch is None: break # feed batch into network feed_dict = { image_pl: img_batch, pos_pcl_pl: pcl_batch, is_training: False } img_batch_feature, pcl_batch_feature = sess.run( [image_feature, pos_pcl_feature], feed_dict=feed_dict) img_feature[ batch_counter:batch_counter + img_batch_feature.shape[0], :] = img_batch_feature pcl_feature[ batch_counter:batch_counter + pcl_batch_feature.shape[0], :] = pcl_batch_feature batch_counter += img_batch_feature.shape[0] print(' Compute top 1 accuracy') # compute top1 accuracy and record data val_accuracy = ComputeAccuracy(img_feature, pcl_feature) # with open('tensorboard/' + 'v2_trainlist_accuracy.txt', 'a') as file: # file.write(str(epoch) + ' ' + str(global_step_val) + ' : ' + str(val_accuracy)+ ', model version: '+ save_version+'\n') # print(' global step: %d, accuracy = %.3f%%' % (global_step_val, val_accuracy*100.0)) # --------------------------------------------------------- print val_accuracy # read a batch img_batch, pos_pcl_batch, neg_pcl_batch = data.getTrainBatch() # return None, end of this epoch if img_batch is None: break global_step_val = tf.train.global_step(sess, global_step) feed_dict = { image_pl: img_batch, pos_pcl_pl: pos_pcl_batch, neg_pcl_pl: neg_pcl_batch, learning_rate: learning_rate_val, is_training: True } if num_of_iterations % 20 == 0: _, loss_val = \ sess.run([train_step, loss], feed_dict=feed_dict) print( ' global %d, epoch %d, iter %d: loss: %.4f' % (global_step_val, epoch, num_of_iterations, loss_val)) # tensorboard: add training information #train_writer.add_summary(summary_val, global_step_val) else: sess.run(train_step, feed_dict=feed_dict) # increment number of iterations num_of_iterations += 1