def train(load_version, train_list, test_list): print('----------------- START to train -----------------') new_test_list = test_list[0][1] + test_list[0][2] print new_test_list[1] data = Data(batch_size, image_size, pcl_size, train_list, new_test_list) total_test_num = (len(new_test_list) // batch_size) * batch_size # data = Data(batch_size, image_size, pcl_size, # train_list, test_list[0][1]) # total_test_num = (len(test_list[0][1]) // batch_size) * batch_size # data = Data(batch_size, image_size, pcl_size, # train_list, train_list[0][779:907]) # record test_list for checking # with open('test_list.txt', 'w') as file: # for line in test_list[0][100]: # file.writelines('%s %s %s %s\n' % (line.submap_id,line.cam_id, line.sift_filename,line.iss_filename)) # define placeholder image_pl = tf.placeholder(tf.float32, shape=[batch_size, image_size, image_size, 3]) pos_pcl_pl = tf.placeholder(tf.float32, shape=[batch_size, pcl_size, 3]) neg_pcl_pl = tf.placeholder(tf.float32, shape=[batch_size, pcl_size, 3]) is_training = tf.placeholder(tf.bool) #label_pl = tf.placeholder(tf.int32 , shape=[batch_size, 1]) learning_rate = tf.placeholder(tf.float32) # tensorboard: visualise sift image patch #tf.summary.image('input_sift_image', image_pl, 64) # build model print('build model') with tf.device('/gpu:1'): # use gpu 1 to forward with tf.variable_scope('image_branch') as scope: image_feature = vgg16(image_pl, is_training=True, output_dim=image_feature_dim, bn_decay=None) with tf.variable_scope('pointcloud_branch') as scope: pos_pcl_feature, _ = pointNet(pos_pcl_pl, pcl_feature_dim, is_training=is_training, use_bn=False, bn_decay=None) scope.reuse_variables() neg_pcl_feature, _ = pointNet(neg_pcl_pl, pcl_feature_dim, is_training=is_training, use_bn=False, bn_decay=None) # define loss print('define loss...') loss = triplet_loss(image_feature, pos_pcl_feature, neg_pcl_feature, 0) # tensorboard: visualise loss tf.summary.scalar('loss', loss) # set training print('set training...') with tf.device('/gpu:0'): # use gpu 0 to backward # set global step global_step = tf.Variable(0, trainable=False) # set learning optimisation with tf.name_scope('train'): train_step = tf.train.AdamOptimizer( learning_rate, 0.9, 0.999).minimize(loss, global_step=global_step) saver = tf.train.Saver(tf.all_variables(), max_to_keep=None) # tf.global_variables # run model print('run model...') config = tf.ConfigProto(log_device_placement=False, allow_soft_placement=True) config.gpu_options.allow_growth = True config.gpu_options.per_process_gpu_memory_fraction = 0.9 with tf.Session(config=config) as sess: # summary #print('initialise tensorboard...') # merged = tf.summary.merge_all() save_version = 'v1' # train_writer = tf.summary.FileWriter('tensorboard/' + save_version + '/train', sess.graph) # test_writer = tf.summary.FileWriter('tensorboard/' + save_version + '/test') print('initialise model...') sess.run(tf.global_variables_initializer()) print(' load model...') save_path = 'model/' + 'v1' + '/' + load_version + '_model.ckpt' saver.restore(sess, save_path) #restore_tf_model(sess) print(" Model loaded from: %s" % save_path) # Train and Test global_step_val = 0 for epoch in range(epoch_time): epoch += 31 num_of_iterations = 0 # # --------------------- evaluate model --------------------------- # print('**** Validate ...') # print(' Compute image and pcl descriptors') # # # test the first run /test_list[0] only # # total_test_num = (len(test_list[0][1]) // batch_size) * batch_size # #total_test_num = (len(train_list[0][779:907]) // batch_size) * batch_size # # img_feature = np.zeros([total_test_num, image_feature_dim]) # pcl_feature = np.zeros([total_test_num, pcl_feature_dim]) # # batch_counter = 0 # # # feed test list into network # while True: # # read a batch # img_batch, pcl_batch = data.getTestBatch() # # return None, end of this epoch # if img_batch is None: # break # # # feed batch into network # feed_dict = {image_pl: img_batch, pos_pcl_pl: pcl_batch, is_training: False} # img_batch_feature, pcl_batch_feature = sess.run([image_feature, pos_pcl_feature], feed_dict=feed_dict) # img_feature[batch_counter: batch_counter+img_batch_feature.shape[0],:] = img_batch_feature # pcl_feature[batch_counter: batch_counter+pcl_batch_feature.shape[0],:] = pcl_batch_feature # # batch_counter += img_batch_feature.shape[0] # # print(' Compute top 1 accuracy') # # compute top1 accuracy and record data # val_accuracy = ComputeAccuracy(img_feature, pcl_feature) # with open('tensorboard/' + 'v2_trainlist_accuracy.txt', 'a') as file: # file.write(str(epoch) + ' ' + str(global_step_val) + ' : ' + str(val_accuracy)+'\n') # #file.write(str(epoch) + ' ' + str(global_step_val) + ' : ' + str(val_accuracy)+ ', model version: '+ save_version+'\n') # print(' global step: %d, accuracy = %.3f%%' % (global_step_val, val_accuracy*100.0)) # # ---------------------------------------------------------------- # --------------------- train model ---------------------- # shuffle train list data.train_list = shuffleTrainset(train_list) while True: if num_of_iterations % 2000 == 0: # save model save_version = 'v2_' + str(epoch) + '_' + str( num_of_iterations) save_path = saver.save( sess, 'model/v1' + '/' + save_version + '_model.ckpt') print(" Model saved in file: %s" % save_path) # -------------------- evaluate model --------------------- print('**** Validate ...') print(' Compute image and pcl descriptors') # test the first run /test_lists[0] only #total_test_num = (len(train_list[0][779:907]) // batch_size) * batch_size # total_test_num = (len(test_list[0][1]) // batch_size) * batch_size img_feature = np.zeros([total_test_num, image_feature_dim]) pcl_feature = np.zeros([total_test_num, pcl_feature_dim]) batch_counter = 0 # feed test list into network while True: # read a batch img_batch, pcl_batch = data.getTestBatch() # return None, end of this epoch if img_batch is None: break # feed batch into network feed_dict = { image_pl: img_batch, pos_pcl_pl: pcl_batch, is_training: False } img_batch_feature, pcl_batch_feature = sess.run( [image_feature, pos_pcl_feature], feed_dict=feed_dict) img_feature[ batch_counter:batch_counter + img_batch_feature.shape[0], :] = img_batch_feature pcl_feature[ batch_counter:batch_counter + pcl_batch_feature.shape[0], :] = pcl_batch_feature batch_counter += img_batch_feature.shape[0] print(' Compute top 1 accuracy') # compute top1 accuracy and record data val_accuracy = ComputeAccuracy(img_feature, pcl_feature) # with open('tensorboard/' + 'v2_trainlist_accuracy.txt', 'a') as file: # file.write(str(epoch) + ' ' + str(global_step_val) + ' : ' + str(val_accuracy)+ ', model version: '+ save_version+'\n') # print(' global step: %d, accuracy = %.3f%%' % (global_step_val, val_accuracy*100.0)) # --------------------------------------------------------- print val_accuracy # read a batch img_batch, pos_pcl_batch, neg_pcl_batch = data.getTrainBatch() # return None, end of this epoch if img_batch is None: break global_step_val = tf.train.global_step(sess, global_step) feed_dict = { image_pl: img_batch, pos_pcl_pl: pos_pcl_batch, neg_pcl_pl: neg_pcl_batch, learning_rate: learning_rate_val, is_training: True } if num_of_iterations % 20 == 0: _, loss_val = \ sess.run([train_step, loss], feed_dict=feed_dict) print( ' global %d, epoch %d, iter %d: loss: %.4f' % (global_step_val, epoch, num_of_iterations, loss_val)) # tensorboard: add training information #train_writer.add_summary(summary_val, global_step_val) else: sess.run(train_step, feed_dict=feed_dict) # increment number of iterations num_of_iterations += 1
def test(load_version, sift_test_list, iss_test_list, submap_id): print('----------------- START to test -----------------') #sift_test_list = sift_test_list[submap_id-1][submap_image_id-1] iss_test_list = iss_test_list[submap_id - 1] iss_test_file = "iss_test_list_txt/%03d.txt" % submap_id with open(iss_test_file, 'w') as file: for i in range(len(iss_test_list)): file.write('%s\n' % iss_test_list[i]) # define placeholder image_pl = tf.placeholder(tf.float32, shape=[batch_size, image_size, image_size, 3]) pos_pcl_pl = tf.placeholder(tf.float32, shape=[batch_size, pcl_size, 3]) neg_pcl_pl = tf.placeholder(tf.float32, shape=[batch_size, pcl_size, 3]) is_training = tf.placeholder(tf.bool) # build model print('build model') with tf.device('/gpu:0'): # use gpu 1 to forward with tf.variable_scope('image_branch') as scope: image_feature = vgg16(image_pl, is_training=True, output_dim=image_feature_dim, bn_decay=None) with tf.variable_scope('pointcloud_branch') as scope: pos_pcl_feature, _ = pointNet(pos_pcl_pl, pcl_feature_dim, is_training=is_training, use_bn=False, bn_decay=None) scope.reuse_variables() neg_pcl_feature, _ = pointNet(neg_pcl_pl, pcl_feature_dim, is_training=is_training, use_bn=False, bn_decay=None) saver = tf.train.Saver(tf.all_variables(), max_to_keep=None) # tf.global_variables # run model print('run model...') config = tf.ConfigProto(log_device_placement=False, allow_soft_placement=True) config.gpu_options.allow_growth = True config.gpu_options.per_process_gpu_memory_fraction = 0.9 with tf.Session(config=config) as sess: print('initialise model...') sess.run(tf.global_variables_initializer()) print(' load model...') save_path = 'model/' + 'v2' + '/' + load_version + '_model.ckpt' saver.restore(sess, save_path) #restore_tf_model(sess) print(" Model loaded from: %s" % save_path) # -------------------- evaluate model --------------------- print('**** Validate ...') print(' Compute image and pcl descriptors') iss_batch_num = len(iss_test_list) // batch_size iss_test_num = iss_batch_num * batch_size pcl_feature = np.zeros([iss_test_num, pcl_feature_dim]) # feed iss test list into the network batch_counter = 0 print('-------- test iss --------------') for i in range(iss_batch_num): print(" *** iss progress: %d/%d" % (i, iss_batch_num)) pcl_batch = getISSTestBatch(iss_test_list, i) feed_dict = {pos_pcl_pl: pcl_batch, is_training: False} pcl_batch_feature = sess.run(pos_pcl_feature, feed_dict=feed_dict) pcl_feature[batch_counter:batch_counter + pcl_batch_feature.shape[0], :] = pcl_batch_feature batch_counter += pcl_batch_feature.shape[0] print('---------- test sift ----------') sift_submap_test_list = sift_test_list[submap_id - 1] # all images for k in range(len(sift_submap_test_list)): sift_test_list = sift_submap_test_list[k] # image id: i+1 cam_id = sift_test_list[0].split('/')[-2] # expected 'cam1_xxx' # record test_list for checking sift_test_file = "sift_test_list_txt/%03d_%s.txt" % (submap_id, cam_id) with open(sift_test_file, 'w') as file: for i in range(len(sift_test_list)): file.write('%s\n' % sift_test_list[i]) # test the patches from one image in the submap sift_batch_num = len(sift_test_list) // batch_size sift_test_num = sift_batch_num * batch_size img_feature = np.zeros([sift_test_num, image_feature_dim]) # feed sift test list into the network batch_counter = 0 print(" *** image id: %d/%d" % (k, len(sift_submap_test_list))) for i in range(sift_batch_num): #print(" *** image id: %d/%d, batch id: %d/%d" % (k, len(sift_submap_test_list), i, sift_batch_num)) img_batch = getSIFTTestBatch(sift_test_list, i) #print img_batch.shape feed_dict = {image_pl: img_batch, is_training: False} img_batch_feature = sess.run(image_feature, feed_dict=feed_dict) #print type(img_batch_feature) img_feature[batch_counter:batch_counter + img_batch_feature.shape[0], :] = img_batch_feature batch_counter += img_batch_feature.shape[0] # compute distance array between img_feature and pcl_feature img_vec = np.sum(np.multiply(img_feature, img_feature), axis=1, keepdims=True) pcl_vec = np.sum(np.multiply(pcl_feature, pcl_feature), axis=1, keepdims=True) dist_array = img_vec + np.transpose(pcl_vec) - 2 * np.matmul( img_feature, np.transpose(pcl_feature)) print(" image patch num: %d, submap pcl num: %d" % (dist_array.shape[0], dist_array.shape[1])) # find correspondences and record # img_pcl_correspondences = []; cam_id = sift_test_list[0].split('/')[-2] txt_folder = "%s/%03d" % (sift_iss_correspond_dir, submap_id) if not os.path.exists(txt_folder): os.makedirs(txt_folder) txt_file_path = "%s/%s.txt" % (txt_folder, cam_id) top_k = 10 with open(txt_file_path, "w") as file: for i in range(dist_array.shape[0]): #min_dist_id = np.argmin(dist_array[i,:]) min_dist_id = np.argsort(dist_array[i, :])[:top_k] idx = np.concatenate((np.array([i + 1]), min_dist_id + 1)) #print(idx) idx = idx.reshape(1, idx.shape[0]) np.savetxt(file, idx, fmt='%d')
def test(load_version, sift_test_list, iss_test_list, submap_id, cam_id, submap_image_id): print('----------------- START to test -----------------') sift_test_list = sift_test_list[submap_id - 1][submap_image_id - 1] iss_test_list = iss_test_list[submap_id - 1] # record test_list for checking with open('sift_test_list.txt', 'w') as file: for i in range(len(sift_test_list)): file.write('%s\n' % sift_test_list[i]) with open('iss_test_list.txt', 'w') as file: for i in range(len(iss_test_list)): file.write('%s\n' % iss_test_list[i]) # define placeholder image_pl = tf.placeholder(tf.float32, shape=[batch_size, image_size, image_size, 3]) pos_pcl_pl = tf.placeholder(tf.float32, shape=[batch_size, pcl_size, 3]) neg_pcl_pl = tf.placeholder(tf.float32, shape=[batch_size, pcl_size, 3]) is_training = tf.placeholder(tf.bool) # build model print('build model') with tf.device('/gpu:1'): # use gpu 1 to forward with tf.variable_scope('image_branch') as scope: image_feature = vgg16(image_pl, is_training=True, output_dim=image_feature_dim, bn_decay=None) with tf.variable_scope('pointcloud_branch') as scope: pos_pcl_feature, _ = pointNet(pos_pcl_pl, pcl_feature_dim, is_training=is_training, use_bn=False, bn_decay=None) scope.reuse_variables() neg_pcl_feature, _ = pointNet(neg_pcl_pl, pcl_feature_dim, is_training=is_training, use_bn=False, bn_decay=None) saver = tf.train.Saver(tf.all_variables(), max_to_keep=None) # tf.global_variables # run model print('run model...') config = tf.ConfigProto(log_device_placement=False, allow_soft_placement=True) config.gpu_options.allow_growth = True config.gpu_options.per_process_gpu_memory_fraction = 0.9 with tf.Session(config=config) as sess: print('initialise model...') sess.run(tf.global_variables_initializer()) print(' load model...') save_path = 'model/' + 'v1' + '/' + load_version + '_model.ckpt' saver.restore(sess, save_path) #restore_tf_model(sess) print(" Model loaded from: %s" % save_path) # -------------------- evaluate model --------------------- print('**** Validate ...') print(' Compute image and pcl descriptors') # test list sift_batch_num = len(sift_test_list) // batch_size sift_test_num = sift_batch_num * batch_size iss_batch_num = len(iss_test_list) // batch_size iss_test_num = iss_batch_num * batch_size img_feature = np.zeros([sift_test_num, image_feature_dim]) pcl_feature = np.zeros([iss_test_num, pcl_feature_dim]) # feed sift test list into the network batch_counter = 0 print('---------- test sift ----------') for i in range(sift_batch_num): print(" *** sift progress: %d/%d" % (i, sift_batch_num)) img_batch = getSIFTTestBatch(sift_test_list, i) #print img_batch.shape feed_dict = {image_pl: img_batch, is_training: False} img_batch_feature = sess.run(image_feature, feed_dict=feed_dict) #print type(img_batch_feature) img_feature[batch_counter:batch_counter + img_batch_feature.shape[0], :] = img_batch_feature batch_counter += img_batch_feature.shape[0] # feed iss test list into the network batch_counter = 0 print('-------- test iss --------------') for i in range(iss_batch_num): print(" *** iss progress: %d/%d" % (i, iss_batch_num)) pcl_batch = getISSTestBatch(iss_test_list, i) feed_dict = {pos_pcl_pl: pcl_batch, is_training: False} pcl_batch_feature = sess.run(pos_pcl_feature, feed_dict=feed_dict) pcl_feature[batch_counter:batch_counter + pcl_batch_feature.shape[0], :] = pcl_batch_feature batch_counter += pcl_batch_feature.shape[0] # compute distance array between img_feature and pcl_feature img_vec = np.sum(np.multiply(img_feature, img_feature), axis=1, keepdims=True) pcl_vec = np.sum(np.multiply(pcl_feature, pcl_feature), axis=1, keepdims=True) dist_array = img_vec + np.transpose(pcl_vec) - 2 * np.matmul( img_feature, np.transpose(pcl_feature)) print(" image patch num: %d, submap pcl num: %d" % (dist_array.shape[0], dist_array.shape[1])) # find correspondences and record img_pcl_correspondences = [] txt_file_path = "%s/%03d_cam%d_%03d.txt" % ( sift_iss_correspond_dir, submap_id, cam_id, submap_image_id) with open(txt_file_path, "w") as file: for i in range(dist_array.shape[0]): min_dist_id = np.argmin(dist_array[i, :]) min_dist_val = dist_array[i, min_dist_id] #print min_dist_val if min_dist_val <= thresh_dist: img_pcl_correspondences.append( [sift_test_list[i], iss_test_list[min_dist_id]]) file.write('%d %d %s %s\n' % ((i + 1), (min_dist_id + 1), sift_test_list[i], iss_test_list[min_dist_id]))