def train(load_version, train_list, test_list):
print('----------------- START to train -----------------')
new_test_list = test_list[0][1] + test_list[0][2]
print new_test_list[1]
data = Data(batch_size, image_size, pcl_size, train_list, new_test_list)
total_test_num = (len(new_test_list) // batch_size) * batch_size
# data = Data(batch_size, image_size, pcl_size,
# train_list, test_list[0][1])
# total_test_num = (len(test_list[0][1]) // batch_size) * batch_size
# data = Data(batch_size, image_size, pcl_size,
# train_list, train_list[0][779:907])
# record test_list for checking
# with open('test_list.txt', 'w') as file:
# for line in test_list[0][100]:
# file.writelines('%s %s %s %s\n' % (line.submap_id,line.cam_id, line.sift_filename,line.iss_filename))
# define placeholder
image_pl = tf.placeholder(tf.float32,
shape=[batch_size, image_size, image_size, 3])
pos_pcl_pl = tf.placeholder(tf.float32, shape=[batch_size, pcl_size, 3])
neg_pcl_pl = tf.placeholder(tf.float32, shape=[batch_size, pcl_size, 3])
is_training = tf.placeholder(tf.bool)
#label_pl = tf.placeholder(tf.int32 , shape=[batch_size, 1])
learning_rate = tf.placeholder(tf.float32)
# tensorboard: visualise sift image patch
#tf.summary.image('input_sift_image', image_pl, 64)
# build model
print('build model')
with tf.device('/gpu:1'): # use gpu 1 to forward
with tf.variable_scope('image_branch') as scope:
image_feature = vgg16(image_pl,
is_training=True,
output_dim=image_feature_dim,
bn_decay=None)
with tf.variable_scope('pointcloud_branch') as scope:
pos_pcl_feature, _ = pointNet(pos_pcl_pl,
pcl_feature_dim,
is_training=is_training,
use_bn=False,
bn_decay=None)
scope.reuse_variables()
neg_pcl_feature, _ = pointNet(neg_pcl_pl,
pcl_feature_dim,
is_training=is_training,
use_bn=False,
bn_decay=None)
# define loss
print('define loss...')
loss = triplet_loss(image_feature, pos_pcl_feature, neg_pcl_feature, 0)
# tensorboard: visualise loss
tf.summary.scalar('loss', loss)
# set training
print('set training...')
with tf.device('/gpu:0'): # use gpu 0 to backward
# set global step
global_step = tf.Variable(0, trainable=False)
# set learning optimisation
with tf.name_scope('train'):
train_step = tf.train.AdamOptimizer(
learning_rate, 0.9, 0.999).minimize(loss,
global_step=global_step)
saver = tf.train.Saver(tf.all_variables(),
max_to_keep=None) # tf.global_variables
# run model
print('run model...')
config = tf.ConfigProto(log_device_placement=False,
allow_soft_placement=True)
config.gpu_options.allow_growth = True
config.gpu_options.per_process_gpu_memory_fraction = 0.9
with tf.Session(config=config) as sess:
# summary
#print('initialise tensorboard...')
# merged = tf.summary.merge_all()
save_version = 'v1'
# train_writer = tf.summary.FileWriter('tensorboard/' + save_version + '/train', sess.graph)
# test_writer = tf.summary.FileWriter('tensorboard/' + save_version + '/test')
print('initialise model...')
sess.run(tf.global_variables_initializer())
print(' load model...')
save_path = 'model/' + 'v1' + '/' + load_version + '_model.ckpt'
saver.restore(sess, save_path)
#restore_tf_model(sess)
print(" Model loaded from: %s" % save_path)
# Train and Test
global_step_val = 0
for epoch in range(epoch_time):
epoch += 31
num_of_iterations = 0
# # --------------------- evaluate model ---------------------------
# print('**** Validate ...')
# print(' Compute image and pcl descriptors')
#
# # test the first run /test_list[0] only
#
# total_test_num = (len(test_list[0][1]) // batch_size) * batch_size
# #total_test_num = (len(train_list[0][779:907]) // batch_size) * batch_size
#
# img_feature = np.zeros([total_test_num, image_feature_dim])
# pcl_feature = np.zeros([total_test_num, pcl_feature_dim])
#
# batch_counter = 0
#
# # feed test list into network
# while True:
# # read a batch
# img_batch, pcl_batch = data.getTestBatch()
# # return None, end of this epoch
# if img_batch is None:
# break
#
# # feed batch into network
# feed_dict = {image_pl: img_batch, pos_pcl_pl: pcl_batch, is_training: False}
# img_batch_feature, pcl_batch_feature = sess.run([image_feature, pos_pcl_feature], feed_dict=feed_dict)
# img_feature[batch_counter: batch_counter+img_batch_feature.shape[0],:] = img_batch_feature
# pcl_feature[batch_counter: batch_counter+pcl_batch_feature.shape[0],:] = pcl_batch_feature
#
# batch_counter += img_batch_feature.shape[0]
#
# print(' Compute top 1 accuracy')
# # compute top1 accuracy and record data
# val_accuracy = ComputeAccuracy(img_feature, pcl_feature)
# with open('tensorboard/' + 'v2_trainlist_accuracy.txt', 'a') as file:
# file.write(str(epoch) + ' ' + str(global_step_val) + ' : ' + str(val_accuracy)+'\n')
# #file.write(str(epoch) + ' ' + str(global_step_val) + ' : ' + str(val_accuracy)+ ', model version: '+ save_version+'\n')
# print(' global step: %d, accuracy = %.3f%%' % (global_step_val, val_accuracy*100.0))
# # ----------------------------------------------------------------
# --------------------- train model ----------------------
# shuffle train list
data.train_list = shuffleTrainset(train_list)
while True:
if num_of_iterations % 2000 == 0:
# save model
save_version = 'v2_' + str(epoch) + '_' + str(
num_of_iterations)
save_path = saver.save(
sess, 'model/v1' + '/' + save_version + '_model.ckpt')
print(" Model saved in file: %s" % save_path)
# -------------------- evaluate model ---------------------
print('**** Validate ...')
print(' Compute image and pcl descriptors')
# test the first run /test_lists[0] only
#total_test_num = (len(train_list[0][779:907]) // batch_size) * batch_size
# total_test_num = (len(test_list[0][1]) // batch_size) * batch_size
img_feature = np.zeros([total_test_num, image_feature_dim])
pcl_feature = np.zeros([total_test_num, pcl_feature_dim])
batch_counter = 0
# feed test list into network
while True:
# read a batch
img_batch, pcl_batch = data.getTestBatch()
# return None, end of this epoch
if img_batch is None:
break
# feed batch into network
feed_dict = {
image_pl: img_batch,
pos_pcl_pl: pcl_batch,
is_training: False
}
img_batch_feature, pcl_batch_feature = sess.run(
[image_feature, pos_pcl_feature],
feed_dict=feed_dict)
img_feature[
batch_counter:batch_counter +
img_batch_feature.shape[0], :] = img_batch_feature
pcl_feature[
batch_counter:batch_counter +
pcl_batch_feature.shape[0], :] = pcl_batch_feature
batch_counter += img_batch_feature.shape[0]
print(' Compute top 1 accuracy')
# compute top1 accuracy and record data
val_accuracy = ComputeAccuracy(img_feature, pcl_feature)
# with open('tensorboard/' + 'v2_trainlist_accuracy.txt', 'a') as file:
# file.write(str(epoch) + ' ' + str(global_step_val) + ' : ' + str(val_accuracy)+ ', model version: '+ save_version+'\n')
# print(' global step: %d, accuracy = %.3f%%' % (global_step_val, val_accuracy*100.0))
# ---------------------------------------------------------
print val_accuracy
# read a batch
img_batch, pos_pcl_batch, neg_pcl_batch = data.getTrainBatch()
# return None, end of this epoch
if img_batch is None:
break
global_step_val = tf.train.global_step(sess, global_step)
feed_dict = {
image_pl: img_batch,
pos_pcl_pl: pos_pcl_batch,
neg_pcl_pl: neg_pcl_batch,
learning_rate: learning_rate_val,
is_training: True
}
if num_of_iterations % 20 == 0:
_, loss_val = \
sess.run([train_step, loss], feed_dict=feed_dict)
print(
' global %d, epoch %d, iter %d: loss: %.4f' %
(global_step_val, epoch, num_of_iterations, loss_val))
# tensorboard: add training information
#train_writer.add_summary(summary_val, global_step_val)
else:
sess.run(train_step, feed_dict=feed_dict)
# increment number of iterations
num_of_iterations += 1
def test(load_version, sift_test_list, iss_test_list, submap_id):
print('----------------- START to test -----------------')
#sift_test_list = sift_test_list[submap_id-1][submap_image_id-1]
iss_test_list = iss_test_list[submap_id - 1]
iss_test_file = "iss_test_list_txt/%03d.txt" % submap_id
with open(iss_test_file, 'w') as file:
for i in range(len(iss_test_list)):
file.write('%s\n' % iss_test_list[i])
# define placeholder
image_pl = tf.placeholder(tf.float32,
shape=[batch_size, image_size, image_size, 3])
pos_pcl_pl = tf.placeholder(tf.float32, shape=[batch_size, pcl_size, 3])
neg_pcl_pl = tf.placeholder(tf.float32, shape=[batch_size, pcl_size, 3])
is_training = tf.placeholder(tf.bool)
# build model
print('build model')
with tf.device('/gpu:0'): # use gpu 1 to forward
with tf.variable_scope('image_branch') as scope:
image_feature = vgg16(image_pl,
is_training=True,
output_dim=image_feature_dim,
bn_decay=None)
with tf.variable_scope('pointcloud_branch') as scope:
pos_pcl_feature, _ = pointNet(pos_pcl_pl,
pcl_feature_dim,
is_training=is_training,
use_bn=False,
bn_decay=None)
scope.reuse_variables()
neg_pcl_feature, _ = pointNet(neg_pcl_pl,
pcl_feature_dim,
is_training=is_training,
use_bn=False,
bn_decay=None)
saver = tf.train.Saver(tf.all_variables(),
max_to_keep=None) # tf.global_variables
# run model
print('run model...')
config = tf.ConfigProto(log_device_placement=False,
allow_soft_placement=True)
config.gpu_options.allow_growth = True
config.gpu_options.per_process_gpu_memory_fraction = 0.9
with tf.Session(config=config) as sess:
print('initialise model...')
sess.run(tf.global_variables_initializer())
print(' load model...')
save_path = 'model/' + 'v2' + '/' + load_version + '_model.ckpt'
saver.restore(sess, save_path)
#restore_tf_model(sess)
print(" Model loaded from: %s" % save_path)
# -------------------- evaluate model ---------------------
print('**** Validate ...')
print(' Compute image and pcl descriptors')
iss_batch_num = len(iss_test_list) // batch_size
iss_test_num = iss_batch_num * batch_size
pcl_feature = np.zeros([iss_test_num, pcl_feature_dim])
# feed iss test list into the network
batch_counter = 0
print('-------- test iss --------------')
for i in range(iss_batch_num):
print(" *** iss progress: %d/%d" % (i, iss_batch_num))
pcl_batch = getISSTestBatch(iss_test_list, i)
feed_dict = {pos_pcl_pl: pcl_batch, is_training: False}
pcl_batch_feature = sess.run(pos_pcl_feature, feed_dict=feed_dict)
pcl_feature[batch_counter:batch_counter +
pcl_batch_feature.shape[0], :] = pcl_batch_feature
batch_counter += pcl_batch_feature.shape[0]
print('---------- test sift ----------')
sift_submap_test_list = sift_test_list[submap_id - 1] # all images
for k in range(len(sift_submap_test_list)):
sift_test_list = sift_submap_test_list[k] # image id: i+1
cam_id = sift_test_list[0].split('/')[-2] # expected 'cam1_xxx'
# record test_list for checking
sift_test_file = "sift_test_list_txt/%03d_%s.txt" % (submap_id,
cam_id)
with open(sift_test_file, 'w') as file:
for i in range(len(sift_test_list)):
file.write('%s\n' % sift_test_list[i])
# test the patches from one image in the submap
sift_batch_num = len(sift_test_list) // batch_size
sift_test_num = sift_batch_num * batch_size
img_feature = np.zeros([sift_test_num, image_feature_dim])
# feed sift test list into the network
batch_counter = 0
print(" *** image id: %d/%d" % (k, len(sift_submap_test_list)))
for i in range(sift_batch_num):
#print(" *** image id: %d/%d, batch id: %d/%d" % (k, len(sift_submap_test_list), i, sift_batch_num))
img_batch = getSIFTTestBatch(sift_test_list, i)
#print img_batch.shape
feed_dict = {image_pl: img_batch, is_training: False}
img_batch_feature = sess.run(image_feature,
feed_dict=feed_dict)
#print type(img_batch_feature)
img_feature[batch_counter:batch_counter +
img_batch_feature.shape[0], :] = img_batch_feature
batch_counter += img_batch_feature.shape[0]
# compute distance array between img_feature and pcl_feature
img_vec = np.sum(np.multiply(img_feature, img_feature),
axis=1,
keepdims=True)
pcl_vec = np.sum(np.multiply(pcl_feature, pcl_feature),
axis=1,
keepdims=True)
dist_array = img_vec + np.transpose(pcl_vec) - 2 * np.matmul(
img_feature, np.transpose(pcl_feature))
print(" image patch num: %d, submap pcl num: %d" %
(dist_array.shape[0], dist_array.shape[1]))
# find correspondences and record
# img_pcl_correspondences = [];
cam_id = sift_test_list[0].split('/')[-2]
txt_folder = "%s/%03d" % (sift_iss_correspond_dir, submap_id)
if not os.path.exists(txt_folder):
os.makedirs(txt_folder)
txt_file_path = "%s/%s.txt" % (txt_folder, cam_id)
top_k = 10
with open(txt_file_path, "w") as file:
for i in range(dist_array.shape[0]):
#min_dist_id = np.argmin(dist_array[i,:])
min_dist_id = np.argsort(dist_array[i, :])[:top_k]
idx = np.concatenate((np.array([i + 1]), min_dist_id + 1))
#print(idx)
idx = idx.reshape(1, idx.shape[0])
np.savetxt(file, idx, fmt='%d')
def test(load_version, sift_test_list, iss_test_list, submap_id, cam_id,
submap_image_id):
print('----------------- START to test -----------------')
sift_test_list = sift_test_list[submap_id - 1][submap_image_id - 1]
iss_test_list = iss_test_list[submap_id - 1]
# record test_list for checking
with open('sift_test_list.txt', 'w') as file:
for i in range(len(sift_test_list)):
file.write('%s\n' % sift_test_list[i])
with open('iss_test_list.txt', 'w') as file:
for i in range(len(iss_test_list)):
file.write('%s\n' % iss_test_list[i])
# define placeholder
image_pl = tf.placeholder(tf.float32,
shape=[batch_size, image_size, image_size, 3])
pos_pcl_pl = tf.placeholder(tf.float32, shape=[batch_size, pcl_size, 3])
neg_pcl_pl = tf.placeholder(tf.float32, shape=[batch_size, pcl_size, 3])
is_training = tf.placeholder(tf.bool)
# build model
print('build model')
with tf.device('/gpu:1'): # use gpu 1 to forward
with tf.variable_scope('image_branch') as scope:
image_feature = vgg16(image_pl,
is_training=True,
output_dim=image_feature_dim,
bn_decay=None)
with tf.variable_scope('pointcloud_branch') as scope:
pos_pcl_feature, _ = pointNet(pos_pcl_pl,
pcl_feature_dim,
is_training=is_training,
use_bn=False,
bn_decay=None)
scope.reuse_variables()
neg_pcl_feature, _ = pointNet(neg_pcl_pl,
pcl_feature_dim,
is_training=is_training,
use_bn=False,
bn_decay=None)
saver = tf.train.Saver(tf.all_variables(),
max_to_keep=None) # tf.global_variables
# run model
print('run model...')
config = tf.ConfigProto(log_device_placement=False,
allow_soft_placement=True)
config.gpu_options.allow_growth = True
config.gpu_options.per_process_gpu_memory_fraction = 0.9
with tf.Session(config=config) as sess:
print('initialise model...')
sess.run(tf.global_variables_initializer())
print(' load model...')
save_path = 'model/' + 'v1' + '/' + load_version + '_model.ckpt'
saver.restore(sess, save_path)
#restore_tf_model(sess)
print(" Model loaded from: %s" % save_path)
# -------------------- evaluate model ---------------------
print('**** Validate ...')
print(' Compute image and pcl descriptors')
# test list
sift_batch_num = len(sift_test_list) // batch_size
sift_test_num = sift_batch_num * batch_size
iss_batch_num = len(iss_test_list) // batch_size
iss_test_num = iss_batch_num * batch_size
img_feature = np.zeros([sift_test_num, image_feature_dim])
pcl_feature = np.zeros([iss_test_num, pcl_feature_dim])
# feed sift test list into the network
batch_counter = 0
print('---------- test sift ----------')
for i in range(sift_batch_num):
print(" *** sift progress: %d/%d" % (i, sift_batch_num))
img_batch = getSIFTTestBatch(sift_test_list, i)
#print img_batch.shape
feed_dict = {image_pl: img_batch, is_training: False}
img_batch_feature = sess.run(image_feature, feed_dict=feed_dict)
#print type(img_batch_feature)
img_feature[batch_counter:batch_counter +
img_batch_feature.shape[0], :] = img_batch_feature
batch_counter += img_batch_feature.shape[0]
# feed iss test list into the network
batch_counter = 0
print('-------- test iss --------------')
for i in range(iss_batch_num):
print(" *** iss progress: %d/%d" % (i, iss_batch_num))
pcl_batch = getISSTestBatch(iss_test_list, i)
feed_dict = {pos_pcl_pl: pcl_batch, is_training: False}
pcl_batch_feature = sess.run(pos_pcl_feature, feed_dict=feed_dict)
pcl_feature[batch_counter:batch_counter +
pcl_batch_feature.shape[0], :] = pcl_batch_feature
batch_counter += pcl_batch_feature.shape[0]
# compute distance array between img_feature and pcl_feature
img_vec = np.sum(np.multiply(img_feature, img_feature),
axis=1,
keepdims=True)
pcl_vec = np.sum(np.multiply(pcl_feature, pcl_feature),
axis=1,
keepdims=True)
dist_array = img_vec + np.transpose(pcl_vec) - 2 * np.matmul(
img_feature, np.transpose(pcl_feature))
print(" image patch num: %d, submap pcl num: %d" %
(dist_array.shape[0], dist_array.shape[1]))
# find correspondences and record
img_pcl_correspondences = []
txt_file_path = "%s/%03d_cam%d_%03d.txt" % (
sift_iss_correspond_dir, submap_id, cam_id, submap_image_id)
with open(txt_file_path, "w") as file:
for i in range(dist_array.shape[0]):
min_dist_id = np.argmin(dist_array[i, :])
min_dist_val = dist_array[i, min_dist_id]
#print min_dist_val
if min_dist_val <= thresh_dist:
img_pcl_correspondences.append(
[sift_test_list[i], iss_test_list[min_dist_id]])
file.write('%d %d %s %s\n' %
((i + 1), (min_dist_id + 1), sift_test_list[i],
iss_test_list[min_dist_id]))