def estimate(area):
LOG_DIR = 'log{}'.format(area)
num_classes = 13
file_path = "data/train_hdf5_file_list_woArea{}.txt".format(area)
train_file_list = provider.getDataFiles(file_path)
mean_ins_size = np.zeros(num_classes)
ptsnum_in_gt = [[] for itmp in range(num_classes)]
train_data = []
train_group = []
train_sem = []
for h5_filename in train_file_list:
cur_data, cur_group, _, cur_sem = provider.loadDataFile_with_groupseglabel_stanfordindoor(h5_filename)
cur_data = np.reshape(cur_data, [-1, cur_data.shape[-1]])
cur_group = np.reshape(cur_group, [-1])
cur_sem = np.reshape(cur_sem, [-1])
un = np.unique(cur_group)
for ig, g in enumerate(un):
tmp = (cur_group == g)
sem_seg_g = int(stats.mode(cur_sem[tmp])[0])
ptsnum_in_gt[sem_seg_g].append(np.sum(tmp))
for idx in range(num_classes):
mean_ins_size[idx] = np.mean(ptsnum_in_gt[idx]).astype(np.int)
print(mean_ins_size)
np.savetxt(os.path.join(LOG_DIR, 'mean_ins_size.txt'),mean_ins_size)
def estimate(flags):
num_classes = flags.num_cls
if flags.dataset == 'S3DIS':
train_file_list = [os.path.join(flags.data_root, line.strip()) for line in open(flags.input_list, 'r')]
else:
print("Error: Not support the dataset: ", flags.dataset)
return
mean_ins_size = np.zeros(num_classes)
ptsnum_in_gt = [[] for itmp in range(num_classes)]
for h5_filename in train_file_list:
print(h5_filename)
cur_data, cur_group, _, cur_sem = provider.loadDataFile_with_groupseglabel_stanfordindoor(h5_filename)
cur_data = np.reshape(cur_data, [-1, cur_data.shape[-1]])
cur_group = np.reshape(cur_group, [-1])
cur_sem = np.reshape(cur_sem, [-1])
un = np.unique(cur_group)
for ig, g in enumerate(un):
tmp = (cur_group == g)
sem_seg_g = int(stats.mode(cur_sem[tmp])[0])
ptsnum_in_gt[sem_seg_g].append(np.sum(tmp))
for idx in range(num_classes):
mean_ins_size[idx] = np.mean(ptsnum_in_gt[idx]).astype(np.int)
print(mean_ins_size)
np.savetxt(os.path.join(flags.out_dir, 'mean_ins_size.txt'), mean_ins_size)
def estimate_mean_thresholds():
with tf.Graph().as_default():
with tf.device('/gpu:' + str(GPU_INDEX)):
pointclouds_pl, labels_pl, sem_labels_pl = placeholder_inputs(BATCH_SIZE, NUM_POINT)
is_training_pl = tf.placeholder(tf.bool, shape=())
# Get model
pred_sem, pred_ins = get_model(pointclouds_pl, is_training_pl, NUM_CLASSES)
loader = tf.train.Saver()
# Create a session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
config.log_device_placement = True
sess = tf.Session(config=config)
is_training = False
# Restore all the variables
loader.restore(sess, MODEL_PATH)
print("Model restored")
train_file_list = provider.getDataFiles(ESTIMATE_FILE_PATH)
ths = np.zeros(NUM_CLASSES)
cnt = np.zeros(NUM_CLASSES)
ths_ = np.zeros(NUM_CLASSES)
for shape_idx in range(len(train_file_list)):
h5_filename = train_file_list[shape_idx]
print('Loading file ' + h5_filename)
cur_data, cur_group, _, cur_sem = provider.loadDataFile_with_groupseglabel_stanfordindoor(h5_filename)
num_data = cur_data.shape[0]
for j in range(num_data):
print("Processing: Shape [%d] Block[%d]" % (shape_idx, j))
pts = cur_data[j, ...]
seg = cur_sem[j, ...]
ins = cur_group[j, ...]
feed_dict = {
pointclouds_pl: np.expand_dims(pts, 0),
is_training_pl: is_training
}
pred_ins_val = sess.run([pred_ins], feed_dict=feed_dict)
pred_ins_val = np.squeeze(pred_ins_val, axis=0)
dis_mat = np.expand_dims(pred_ins_val, axis=1) - np.expand_dims(pred_ins_val, axis=0)
dis_mat = np.linalg.norm(dis_mat, ord=1, axis=2)
ths, ths_, cnt = Get_Ths(dis_mat, seg, ins, ths, ths_, cnt)
print(cnt)
ths = [ths[i] / cnt[i] if cnt[i] != 0 else 0.2 for i in range(len(cnt))]
print(ths)
np.savetxt(os.path.join(LOG_DIR, 'mean_thresholds.txt'), ths)
def load_data_file(input_file):
cur_data, cur_group, _, cur_sem = provider.loadDataFile_with_groupseglabel_stanfordindoor(
input_file)
return cur_data, cur_sem, cur_group
def predict():
is_training = False
with tf.device('/gpu:' + str(gpu_to_use)):
is_training_ph = tf.placeholder(tf.bool, shape=())
pointclouds_ph, ptsseglabel_ph, ptsgroup_label_ph, _, _, _ = \
model.placeholder_inputs(BATCH_SIZE, POINT_NUM, NUM_GROUPS, NUM_CATEGORY)
group_mat_label = tf.matmul(
ptsgroup_label_ph, tf.transpose(ptsgroup_label_ph, perm=[0, 2, 1]))
net_output = model.get_model(pointclouds_ph,
is_training_ph,
group_cate_num=NUM_CATEGORY)
# Add ops to save and restore all the variables.
saver = tf.train.Saver()
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
with tf.Session(config=config) as sess:
# Restore variables from disk.
ckptstate = tf.train.get_checkpoint_state(PRETRAINED_MODEL_PATH)
if ckptstate is not None:
LOAD_MODEL_FILE = os.path.join(
PRETRAINED_MODEL_PATH,
os.path.basename(ckptstate.model_checkpoint_path))
saver.restore(sess, LOAD_MODEL_FILE)
print("Model loaded in file: %s" % LOAD_MODEL_FILE)
else:
print("Fail to load modelfile: %s" % PRETRAINED_MODEL_PATH)
ths = np.zeros(NUM_CATEGORY)
ths_ = np.zeros(NUM_CATEGORY)
cnt = np.zeros(NUM_CATEGORY)
min_groupsize = np.zeros(NUM_CATEGORY)
min_groupsize_cnt = np.zeros(NUM_CATEGORY)
for shape_idx in range(len_pts_files):
cur_train_filename = test_file_list[shape_idx]
if not os.path.exists(cur_train_filename):
continue
cur_data, cur_group, _, cur_seg = provider.loadDataFile_with_groupseglabel_stanfordindoor(
cur_train_filename)
if OUTPUT_VERBOSE:
pts = np.reshape(cur_data, [-1, 9])
output_point_cloud_rgb(
pts[:, 6:], pts[:, 3:6],
os.path.join(OUTPUT_DIR, '%d_pts.obj' % (shape_idx)))
pts_label_one_hot, pts_label_mask = model.convert_seg_to_one_hot(
cur_seg)
pts_group_label, _ = model.convert_groupandcate_to_one_hot(
cur_group)
num_data = cur_data.shape[0]
cur_seg_flatten = np.reshape(cur_seg, [-1])
un, indices = np.unique(cur_group, return_index=True)
for iu, u in enumerate(un):
groupsize = np.sum(cur_group == u)
groupcate = cur_seg_flatten[indices[iu]]
min_groupsize[groupcate] += groupsize
# print groupsize, min_groupsize[groupcate]/min_groupsize_cnt[groupcate]
min_groupsize_cnt[groupcate] += 1
for j in range(num_data):
print("Processsing: Shape [%d] Block[%d]" % (shape_idx, j))
pts = cur_data[j, ...]
feed_dict = {
pointclouds_ph: np.expand_dims(pts, 0),
ptsseglabel_ph: np.expand_dims(pts_label_one_hot[j, ...],
0),
ptsgroup_label_ph: np.expand_dims(pts_group_label[j, ...],
0),
is_training_ph: is_training,
}
pts_corr_val0, pred_confidence_val0, ptsclassification_val0, pts_corr_label_val0 = \
sess.run([net_output['simmat'],
net_output['conf'],
net_output['semseg'],
group_mat_label],
feed_dict=feed_dict)
seg = cur_seg[j, ...]
ins = cur_group[j, ...]
pts_corr_val = np.squeeze(pts_corr_val0[0])
pred_confidence_val = np.squeeze(pred_confidence_val0[0])
ptsclassification_val = np.argmax(np.squeeze(
ptsclassification_val0[0]),
axis=1)
pts_corr_label_val = np.squeeze(1 - pts_corr_label_val0)
seg = np.squeeze(seg)
ins = np.squeeze(ins)
ind = (seg == 8)
pts_corr_val0 = (pts_corr_val > 1.).astype(np.float)
print np.mean(
np.transpose(np.abs(pts_corr_label_val[ind] -
pts_corr_val0[ind]),
axes=[1, 0])[ind])
ths, ths_, cnt = Get_Ths(pts_corr_val, seg, ins, ths, ths_,
cnt)
print ths / cnt
if OUTPUT_VERBOSE:
un, indices = np.unique(ins, return_index=True)
for ii, id in enumerate(indices):
corr = pts_corr_val[id].copy()
output_scale_point_cloud(
pts[:, 6:], np.float32(corr),
os.path.join(
OUTPUT_DIR, '%d_%d_%d_%d_scale.obj' %
(shape_idx, j, un[ii], seg[id])))
corr = pts_corr_label_val[id]
output_scale_point_cloud(
pts[:, 6:], np.float32(corr),
os.path.join(
OUTPUT_DIR, '%d_%d_%d_%d_scalegt.obj' %
(shape_idx, j, un[ii], seg[id])))
output_scale_point_cloud(
pts[:, 6:], np.float32(pred_confidence_val),
os.path.join(OUTPUT_DIR,
'%d_%d_conf.obj' % (shape_idx, j)))
output_color_point_cloud(
pts[:, 6:], ptsclassification_val.astype(np.int32),
os.path.join(OUTPUT_DIR, '%d_seg.obj' % (shape_idx)))
ths = [
ths[i] / cnt[i] if cnt[i] != 0 else 0.2 for i in range(len(cnt))
]
np.savetxt(os.path.join(RESTORE_DIR, 'pergroup_thres.txt'), ths)
min_groupsize = [
int(float(min_groupsize[i]) /
min_groupsize_cnt[i]) if min_groupsize_cnt[i] != 0 else 0
for i in range(len(min_groupsize))
]
np.savetxt(os.path.join(RESTORE_DIR, 'mingroupsize.txt'),
min_groupsize)
BN_INIT_DECAY = 0.5
BN_DECAY_DECAY_RATE = 0.5
#BN_DECAY_DECAY_STEP = float(DECAY_STEP * 2)
BN_DECAY_DECAY_STEP = float(DECAY_STEP)
BN_DECAY_CLIP = 0.99
HOSTNAME = socket.gethostname()
# Load ALL data
train_file_list = provider.getDataFiles(TRAINING_FILE_LIST)
train_data = []
train_group = []
train_sem = []
for h5_filename in train_file_list:
cur_data, cur_group, _, cur_sem = provider.loadDataFile_with_groupseglabel_stanfordindoor(
h5_filename)
train_data.append(cur_data)
train_group.append(cur_group)
train_sem.append(cur_sem)
train_data = np.concatenate(train_data, axis=0)
train_group = np.concatenate(train_group, axis=0)
train_sem = np.concatenate(train_sem, axis=0)
def log_string(out_str):
LOG_FOUT.write(out_str + '\n')
LOG_FOUT.flush()
print(out_str)
def get_learning_rate(batch):
def predict():
is_training = False
with tf.device('/gpu:' + str(gpu_to_use)):
is_training_ph = tf.placeholder(tf.bool, shape=())
pointclouds_ph, ptsseglabel_ph, ptsgroup_label_ph, _, _, _ = \
model.placeholder_inputs(BATCH_SIZE, POINT_NUM, NUM_GROUPS, NUM_CATEGORY)
net_output = model.get_model(pointclouds_ph, is_training_ph, group_cate_num=NUM_CATEGORY)
group_mat_label = tf.matmul(ptsgroup_label_ph, tf.transpose(ptsgroup_label_ph, perm=[0, 2, 1])) #BxNxN: (i,j) if i and j in the same group
# Add ops to save and restore all the variables.
saver = tf.train.Saver()
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
with tf.Session(config=config) as sess:
ckptstate = tf.train.get_checkpoint_state(PRETRAINED_MODEL_PATH)
if ckptstate is not None:
LOAD_MODEL_FILE = os.path.join(PRETRAINED_MODEL_PATH,os.path.basename(ckptstate.model_checkpoint_path))
saver.restore(sess, LOAD_MODEL_FILE)
print("Model loaded in file: %s" % LOAD_MODEL_FILE)
else:
print("Fail to load modelfile: %s" % PRETRAINED_MODEL_PATH)
for shape_idx in range(len_pts_files):
cur_train_filename = test_file_list[shape_idx]
if not os.path.exists(cur_train_filename):
continue
cur_data, cur_group, _, cur_seg = provider.loadDataFile_with_groupseglabel_stanfordindoor(cur_train_filename)
seg_output = np.zeros_like(cur_seg)
segrefine_output = np.zeros_like(cur_seg)
group_output = np.zeros_like(cur_group)
conf_output = np.zeros_like(cur_group).astype(np.float)
pts_label_one_hot, pts_label_mask = model.convert_seg_to_one_hot(cur_seg)
pts_group_label, _ = model.convert_groupandcate_to_one_hot(cur_group)
num_data = cur_data.shape[0]
gap = 5e-3
volume_num = int(1. / gap)+1
volume = -1* np.ones([volume_num,volume_num,volume_num]).astype(np.int32)
volume_seg = -1* np.ones([volume_num,volume_num,volume_num, NUM_CATEGORY]).astype(np.int32)
intersections = np.zeros(NUM_CATEGORY)
unions = np.zeros(NUM_CATEGORY)
print('[%d / %d] Block Number: %d' % (shape_idx, len_pts_files, num_data))
print('Loading train file %s' % (cur_train_filename))
flag = True
for j in range(num_data):
pts = cur_data[j,...]
feed_dict = {
pointclouds_ph: np.expand_dims(pts,0),
ptsseglabel_ph: np.expand_dims(pts_label_one_hot[j,...],0),
ptsgroup_label_ph: np.expand_dims(pts_group_label[j,...],0),
is_training_ph: is_training,
}
pts_corr_val0, pred_confidence_val0, ptsclassification_val0, pts_corr_label_val0 = \
sess.run([net_output['simmat'],
net_output['conf'],
net_output['semseg'],
group_mat_label],
feed_dict=feed_dict)
seg = cur_seg[j,...]
ins = cur_group[j,...]
pts_corr_val = np.squeeze(pts_corr_val0[0]) #NxG
pred_confidence_val = np.squeeze(pred_confidence_val0[0])
ptsclassification_val = np.argmax(np.squeeze(ptsclassification_val0[0]),axis=1)
seg = np.squeeze(seg)
# print label_bin
try:
groupids_block, refineseg, group_seg = GroupMerging_old(pts_corr_val, pred_confidence_val, ptsclassification_val, label_bin) # yolo_to_groupt(pts_corr_val, pts_corr_label_val0[0], seg,t=5)
groupids = BlockMerging(volume, volume_seg, pts[:,6:], groupids_block.astype(np.int32), group_seg, gap)
seg_output[j,:] = ptsclassification_val
segrefine_output[j,:] = refineseg
group_output[j,:] = groupids
conf_output[j,:] = pred_confidence_val
###### Generate Results for Evaluation
basefilename = os.path.basename(cur_train_filename).split('.')[-2]
scene_fn = os.path.join(OUTPUT_DIR, '%s.txt' % basefilename)
f_scene = open(scene_fn, 'w')
scene_gt_fn = os.path.join(GT_DIR, '%s.txt' % basefilename)
group_pred = group_output.reshape(-1)
seg_pred = seg_output.reshape(-1)
conf = conf_output.reshape(-1)
pts = cur_data.reshape([-1, 9])
# filtering
x = (pts[:, 6] / gap).astype(np.int32)
y = (pts[:, 7] / gap).astype(np.int32)
z = (pts[:, 8] / gap).astype(np.int32)
for i in range(group_pred.shape[0]):
if volume[x[i], y[i], z[i]] != -1:
group_pred[i] = volume[x[i], y[i], z[i]]
un = np.unique(group_pred)
pts_in_pred = [[] for itmp in range(NUM_CATEGORY)]
group_pred_final = -1 * np.ones_like(group_pred)
grouppred_cnt = 0
for ig, g in enumerate(un): #each object in prediction
if g == -1:
continue
obj_fn = "predicted_masks/%s_%d.txt" % (basefilename, ig)
tmp = (group_pred == g)
sem_seg_g = int(stats.mode(seg_pred[tmp])[0])
if np.sum(tmp) > 0.25 * min_num_pts_in_group[sem_seg_g]:
pts_in_pred[sem_seg_g] += [tmp]
group_pred_final[tmp] = grouppred_cnt
conf_obj = np.mean(conf[tmp])
grouppred_cnt += 1
f_scene.write("%s %d %f\n" % (obj_fn, sem_seg_g, conf_obj))
np.savetxt(os.path.join(OUTPUT_DIR, obj_fn), tmp.astype(np.int), fmt='%d')
seg_gt = cur_seg.reshape(-1)
group_gt = cur_group.reshape(-1)
groupid_gt = seg_gt * 1000 + group_gt
np.savetxt(scene_gt_fn, groupid_gt.astype(np.int64), fmt='%d')
f_scene.close()
if output_verbose:
output_color_point_cloud(pts[:, 6:], seg_pred.astype(np.int32),
os.path.join(OUTPUT_DIR, '%s_segpred.obj' % (obj_fn)))
output_color_point_cloud(pts[:, 6:], group_pred_final.astype(np.int32),
os.path.join(OUTPUT_DIR, '%s_grouppred.obj' % (obj_fn)))
def test():
with tf.Graph().as_default():
with tf.device('/gpu:' + str(GPU_INDEX)):
pointclouds_pl, labels_pl, sem_labels_pl = placeholder_inputs(
BATCH_SIZE, NUM_POINT)
is_training_pl = tf.placeholder(tf.bool, shape=())
# Get model
pred_sem, pred_ins = get_model(pointclouds_pl, is_training_pl,
NUM_CLASSES)
pred_sem_softmax = tf.nn.softmax(pred_sem)
pred_sem_label = tf.argmax(pred_sem_softmax, axis=2)
loader = tf.train.Saver()
# Create a session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
config.log_device_placement = False
sess = tf.Session(config=config)
is_training = False
# Restore variables from disk.
loader.restore(sess, MODEL_PATH)
logger.info("Model restored from {}".format(MODEL_PATH))
ops = {
'pointclouds_pl': pointclouds_pl,
'labels_pl': labels_pl,
'sem_labels_pl': sem_labels_pl,
'is_training_pl': is_training_pl,
'pred_ins': pred_ins,
'pred_sem_label': pred_sem_label,
'pred_sem_softmax': pred_sem_softmax
}
total_acc = 0.0
total_seen = 0
output_filelist_f = os.path.join(LOG_DIR, 'output_filelist.txt')
fout_out_filelist = []
for shape_idx in range(len_pts_files):
room_path = ROOM_PATH_LIST[shape_idx]
out_data_label_filename = os.path.basename(
room_path)[:-EXT_LEN] + '_pred.txt'
out_data_label_filename = os.path.join(OUTPUT_DIR,
out_data_label_filename)
out_gt_label_filename = os.path.basename(
room_path)[:-EXT_LEN] + '_gt.txt'
out_gt_label_filename = os.path.join(OUTPUT_DIR,
out_gt_label_filename)
fout_data_label = []
fout_gt_label = []
fout_out_filelist.append(out_data_label_filename + '\n')
logger.info('%d / %d ...' % (shape_idx, len_pts_files))
logger.info('Loading file ' + room_path)
size_path = room_path
if FILE_TYPE == 'hdf5':
size_path = size_path.replace('indoor3d_ins_seg_hdf5',
'stanford_indoor3d_ins.sem')
size_path = "{}.npy".format(size_path[:-3])
cur_data, cur_group, _, cur_sem = \
provider.loadDataFile_with_groupseglabel_stanfordindoor(room_path)
elif FILE_TYPE == 'numpy':
cur_data, cur_sem, cur_group = \
indoor3d_util.room2blocks_wrapper_normalized(room_path, NUM_POINT, block_size=1.0, stride=0.5,
random_sample=False, sample_num=None)
cur_data = cur_data[:, 0:NUM_POINT, :]
cur_sem = np.squeeze(cur_sem)
cur_group = np.squeeze(cur_group)
# Get room dimension..
data_label = np.load(size_path)
data = data_label[:, 0:6]
max_room_x = max(data[:, 0])
max_room_y = max(data[:, 1])
max_room_z = max(data[:, 2])
cur_pred_sem = np.zeros_like(cur_sem)
cur_pred_sem_softmax = np.zeros(
[cur_sem.shape[0], cur_sem.shape[1], NUM_CLASSES])
group_output = np.zeros_like(cur_group)
gap = 5e-3
volume_num = int(1. / gap) + 1
volume = -1 * np.ones([volume_num, volume_num, volume_num]).astype(
np.int32)
volume_seg = -1 * np.ones([volume_num, volume_num, volume_num
]).astype(np.int32)
num_data = cur_data.shape[0]
for j in range(num_data):
logger.info("Processsing: Shape [%d] Block[%d]" %
(shape_idx, j))
pts = cur_data[j, ...]
group = cur_group[j]
sem = cur_sem[j]
feed_dict = {
ops['pointclouds_pl']: np.expand_dims(pts, 0),
ops['labels_pl']: np.expand_dims(group, 0),
ops['sem_labels_pl']: np.expand_dims(sem, 0),
ops['is_training_pl']: is_training
}
pred_ins_val, pred_sem_label_val, pred_sem_softmax_val = sess.run(
[
ops['pred_ins'], ops['pred_sem_label'],
ops['pred_sem_softmax']
],
feed_dict=feed_dict)
pred_val = np.squeeze(pred_ins_val, axis=0)
pred_sem = np.squeeze(pred_sem_label_val, axis=0)
pred_sem_softmax = np.squeeze(pred_sem_softmax_val, axis=0)
cur_pred_sem[j, :] = pred_sem
cur_pred_sem_softmax[j, ...] = pred_sem_softmax
# cluster
group_seg = {}
bandwidth = BANDWIDTH
num_clusters, labels, cluster_centers = cluster(
pred_val, bandwidth)
for idx_cluster in range(num_clusters):
tmp = (labels == idx_cluster)
estimated_seg = int(stats.mode(pred_sem[tmp])[0])
group_seg[idx_cluster] = estimated_seg
groupids_block = labels
groupids = BlockMerging(volume, volume_seg, pts[:, 6:],
groupids_block.astype(np.int32),
group_seg, gap)
group_output[j, :] = groupids
total_acc += float(np.sum(pred_sem == sem)) / pred_sem.shape[0]
total_seen += 1
group_pred = group_output.reshape(-1)
seg_pred = cur_pred_sem.reshape(-1)
seg_pred_softmax = cur_pred_sem_softmax.reshape([-1, NUM_CLASSES])
pts = cur_data.reshape([-1, 9])
# filtering
x = (pts[:, 6] / gap).astype(np.int32)
y = (pts[:, 7] / gap).astype(np.int32)
z = (pts[:, 8] / gap).astype(np.int32)
for i in range(group_pred.shape[0]):
if volume[x[i], y[i], z[i]] != -1:
group_pred[i] = volume[x[i], y[i], z[i]]
seg_gt = cur_sem.reshape(-1)
un = np.unique(group_pred)
pts_in_pred = [[] for itmp in range(NUM_CLASSES)]
group_pred_final = -1 * np.ones_like(group_pred)
grouppred_cnt = 0
for ig, g in enumerate(un): # each object in prediction
if g == -1:
continue
tmp = (group_pred == g)
sem_seg_g = int(stats.mode(seg_pred[tmp])[0])
# if np.sum(tmp) > 500:
if np.sum(tmp) > 0.25 * mean_num_pts_in_group[sem_seg_g]:
group_pred_final[tmp] = grouppred_cnt
pts_in_pred[sem_seg_g] += [tmp]
grouppred_cnt += 1
pts[:, 6] *= max_room_x
pts[:, 7] *= max_room_y
pts[:, 8] *= max_room_z
pts[:, 3:6] *= 255.0
ins = group_pred_final.astype(np.int32)
sem = seg_pred.astype(np.int32)
sem_softmax = seg_pred_softmax
sem_gt = seg_gt
ins_gt = cur_group.reshape(-1)
for i in range(pts.shape[0]):
fout_data_label.append(
'%f %f %f %d %d %d %f %d %d\n' %
(pts[i, 6], pts[i, 7], pts[i, 8], pts[i, 3], pts[i, 4],
pts[i, 5], sem_softmax[i, sem[i]], sem[i], ins[i]))
fout_gt_label.append('%d %d\n' % (sem_gt[i], ins_gt[i]))
with open(out_data_label_filename, 'w') as fd:
fd.writelines(fout_data_label)
with open(out_gt_label_filename, 'w') as fd:
fd.writelines(fout_gt_label)
if output_verbose:
# file name
outfile_name = ROOM_PATH_LIST[shape_idx].split(
'/')[-1][:-EXT_LEN]
# Raw Point Cloud
output_point_cloud_rgb(
pts[:, 6:], pts[:, 3:6].astype(np.int32),
os.path.join(VIS_DIR, '{}_raw.obj'.format(outfile_name)))
logger.info('Saving file {}_raw.obj'.format(outfile_name))
# Instance Prediction
output_color_point_cloud(
pts[:, 6:], group_pred_final.astype(np.int32),
os.path.join(VIS_DIR,
'{}_pred_ins.obj'.format(outfile_name)))
logger.info('Saving file {}_pred_ins.obj'.format(outfile_name))
# Semantic Prediction
output_color_point_cloud(
pts[:, 6:], seg_pred.astype(np.int32),
os.path.join(VIS_DIR,
'{}_pred_sem.obj'.format(outfile_name)))
logger.info('Saving file {}_pred_sem.obj'.format(outfile_name))
# Instance Ground Truth
output_color_point_cloud(
pts[:, 6:], ins_gt,
os.path.join(VIS_DIR,
'{}_gt_ins.obj'.format(outfile_name)))
logger.info('Saving file {}_gt_ins.obj'.format(outfile_name))
# Semantic Ground Truth
output_color_point_cloud(
pts[:, 6:], sem_gt,
os.path.join(VIS_DIR,
'{}_gt_sem.obj'.format(outfile_name)))
logger.info('Saving file {}_gt_sem.obj'.format(outfile_name))
with open(output_filelist_f, 'w') as fd:
fd.writelines(fout_out_filelist)
def train():
with tf.Graph().as_default():
with tf.device('/gpu:' + str(FLAGS.gpu)):
batch = tf.Variable(0, trainable=False, name='batch')
learning_rate = tf.train.exponential_decay(
BASE_LEARNING_RATE, # base learning rate
batch * BATCH_SIZE, # global_var indicating the number of steps
DECAY_STEP, # step size
DECAY_RATE, # decay rate
staircase=True # Stair-case or continuous decreasing
)
learning_rate = tf.maximum(learning_rate, LEARNING_RATE_CLIP)
lr_op = tf.summary.scalar('learning_rate', learning_rate)
pointclouds_ph, ptsseglabel_ph, ptsgroup_label_ph, pts_seglabel_mask_ph, pts_group_mask_ph, alpha_ph = \
model.placeholder_inputs(BATCH_SIZE, POINT_NUM, NUM_GROUPS, NUM_CATEGORY)
is_training_ph = tf.placeholder(tf.bool, shape=())
labels = {'ptsgroup': ptsgroup_label_ph,
'semseg': ptsseglabel_ph,
'semseg_mask': pts_seglabel_mask_ph,
'group_mask': pts_group_mask_ph}
net_output = model.get_model(pointclouds_ph, is_training_ph, group_cate_num=NUM_CATEGORY, m=MARGINS[0])
loss, grouperr, same, same_cnt, diff, diff_cnt, pos, pos_cnt = model.get_loss(net_output, labels, alpha_ph, MARGINS)
total_training_loss_ph = tf.placeholder(tf.float32, shape=())
group_err_loss_ph = tf.placeholder(tf.float32, shape=())
total_train_loss_sum_op = tf.summary.scalar('total_training_loss', total_training_loss_ph)
group_err_op = tf.summary.scalar('group_err_loss', group_err_loss_ph)
train_variables = tf.trainable_variables()
trainer = tf.train.AdamOptimizer(learning_rate)
train_op = trainer.minimize(loss, var_list=train_variables, global_step=batch)
loader = tf.train.Saver([v for v in tf.all_variables()#])
if
('conf_logits' not in v.name) and
('Fsim' not in v.name) and
('Fsconf' not in v.name) and
('batch' not in v.name)
])
saver = tf.train.Saver([v for v in tf.all_variables()])
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
train_writer = tf.summary.FileWriter(SUMMARIES_FOLDER + '/train', sess.graph)
import glob
train_file_list= glob.glob(os.path.join(TRAINING_FILE_LIST,'*.h5'))
# train_file_list=provider.getDataFiles(TRAINING_FILE_LIST)
num_train_file = len(train_file_list)
fcmd = open(os.path.join(LOG_STORAGE_PATH, 'cmd.txt'), 'w')
fcmd.write(str(FLAGS))
fcmd.close()
flog = open(os.path.join(LOG_STORAGE_PATH, 'log.txt'), 'w')
ckptstate = tf.train.get_checkpoint_state(PRETRAINED_MODEL_PATH)
if ckptstate is not None:
LOAD_MODEL_FILE = os.path.join(PRETRAINED_MODEL_PATH, os.path.basename(ckptstate.model_checkpoint_path))
loader.restore(sess, LOAD_MODEL_FILE)
printout(flog, "Model loaded in file: %s" % LOAD_MODEL_FILE)
else:
printout(flog, "Fail to load modelfile: %s" % PRETRAINED_MODEL_PATH)
train_file_idx = np.arange(0, len(train_file_list))
np.random.shuffle(train_file_idx)
## load all data into memory
all_data = []
all_group = []
all_seg = []
for i in range(num_train_file):
cur_train_filename = train_file_list[train_file_idx[i]]
# printout(flog, 'Loading train file ' + cur_train_filename)
cur_data, cur_group, _, cur_seg = provider.loadDataFile_with_groupseglabel_stanfordindoor(cur_train_filename)
all_data += [cur_data]
all_group += [cur_group]
all_seg += [cur_seg]
all_data = np.concatenate(all_data,axis=0)
all_group = np.concatenate(all_group,axis=0)
all_seg = np.concatenate(all_seg,axis=0)
num_data = all_data.shape[0]
num_batch = num_data // BATCH_SIZE
def train_one_epoch(epoch_num):
### NOTE: is_training = False: We do not update bn parameters during training due to the small batch size. This requires pre-training PointNet with large batchsize (say 32).
is_training = False
order = np.arange(num_data)
np.random.shuffle(order)
total_loss = 0.0
total_grouperr = 0.0
total_same = 0.0
total_diff = 0.0
total_pos = 0.0
same_cnt0 = 0
for j in range(num_batch):
begidx = j * BATCH_SIZE
endidx = (j + 1) * BATCH_SIZE
pts_label_one_hot, pts_label_mask = model.convert_seg_to_one_hot(all_seg[order[begidx: endidx]])
pts_group_label, pts_group_mask = model.convert_groupandcate_to_one_hot(all_group[order[begidx: endidx]])
feed_dict = {
pointclouds_ph: all_data[order[begidx: endidx], ...],
ptsseglabel_ph: pts_label_one_hot,
ptsgroup_label_ph: pts_group_label,
pts_seglabel_mask_ph: pts_label_mask,
pts_group_mask_ph: pts_group_mask,
is_training_ph: is_training,
alpha_ph: min(10., (float(epoch_num) / 5.) * 2. + 2.),
}
_, loss_val, simmat_val, grouperr_val, same_val, same_cnt_val, diff_val, diff_cnt_val, pos_val, pos_cnt_val = sess.run([train_op, loss, net_output['simmat'], grouperr, same, same_cnt, diff, diff_cnt, pos, pos_cnt], feed_dict=feed_dict)
total_loss += loss_val
total_grouperr += grouperr_val
if diff_cnt_val!=0:
total_diff += (diff_val / diff_cnt_val)
if same_cnt_val > 0:
total_same += same_val / same_cnt_val
same_cnt0 += 1
total_pos += pos_val / pos_cnt_val
if j % 10 == 9:
if same_cnt0!=0:
printout(flog, 'Batch: %d, loss: %f, grouperr: %f, same: %f, diff: %f, pos: %f' % (j, total_loss/10, total_grouperr/10, total_same/same_cnt0, total_diff/10, total_pos/10))
else:
printout(flog, 'Batch: %d, loss: %f, grouperr: %f, same: %f, diff: %f, pos: %f' % (j, total_loss/10, total_grouperr/10, 1e10, total_diff/10, total_pos/10))
lr_sum, batch_sum, train_loss_sum, group_err_sum = sess.run( \
[lr_op, batch, total_train_loss_sum_op, group_err_op], \
feed_dict={total_training_loss_ph: total_loss / 10.,
group_err_loss_ph: total_grouperr / 10., })
train_writer.add_summary(train_loss_sum, batch_sum)
train_writer.add_summary(lr_sum, batch_sum)
train_writer.add_summary(group_err_sum, batch_sum)
total_grouperr = 0.0
total_loss = 0.0
total_diff = 0.0
total_same = 0.0
total_pos = 0.0
same_cnt0 = 0
cp_filename = saver.save(sess,
os.path.join(MODEL_STORAGE_PATH, 'epoch_' + str(epoch_num + 1) + '.ckpt'))
printout(flog, 'Successfully store the checkpoint model into ' + cp_filename)
if not os.path.exists(MODEL_STORAGE_PATH):
os.mkdir(MODEL_STORAGE_PATH)
for epoch in range(TRAINING_EPOCHES):
printout(flog, '\n>>> Training for the epoch %d/%d ...' % (epoch, TRAINING_EPOCHES))
train_file_idx = np.arange(0, len(train_file_list))
np.random.shuffle(train_file_idx)
train_one_epoch(epoch)
flog.flush()
cp_filename = saver.save(sess,
os.path.join(MODEL_STORAGE_PATH, 'epoch_' + str(epoch + 1) + '.ckpt'))
printout(flog, 'Successfully store the checkpoint model into ' + cp_filename)
flog.close()
def predict():
is_training = False
with tf.device('/gpu:' + str(gpu_to_use)):
is_training_ph = tf.placeholder(tf.bool, shape=())
pointclouds_ph, ptsseglabel_ph, ptsgroup_label_ph, _, _, _ = \
model.placeholder_inputs(BATCH_SIZE, POINT_NUM, NUM_GROUPS, NUM_CATEGORY)
net_output = model.get_model(pointclouds_ph, is_training_ph, group_cate_num=NUM_CATEGORY)
group_mat_label = tf.matmul(ptsgroup_label_ph, tf.transpose(ptsgroup_label_ph, perm=[0, 2, 1])) #BxNxN: (i,j) if i and j in the same group
# Add ops to save and restore all the variables.
saver = tf.train.Saver()
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
with tf.Session(config=config) as sess:
flog = open(os.path.join(OUTPUT_DIR, 'log.txt'), 'w')
# Restore variables from disk.
ckptstate = tf.train.get_checkpoint_state(PRETRAINED_MODEL_PATH)
if ckptstate is not None:
LOAD_MODEL_FILE = os.path.join(PRETRAINED_MODEL_PATH,os.path.basename(ckptstate.model_checkpoint_path))
saver.restore(sess, LOAD_MODEL_FILE)
printout(flog, "Model loaded in file: %s" % LOAD_MODEL_FILE)
else:
printout(flog, "Fail to load modelfile: %s" % PRETRAINED_MODEL_PATH)
total_acc = 0.0
total_seen = 0
ious = np.zeros(NUM_CATEGORY)
totalnums = np.zeros(NUM_CATEGORY)
tpsins = [[] for itmp in range(NUM_CATEGORY)]#= np.array([]).reshape(0, NUM_CATEGORY)#np.zeros(NUM_CATEGORY)#
fpsins = [[] for itmp in range(NUM_CATEGORY)]#= np.array([]).reshape(0, NUM_CATEGORY)#np.zeros(NUM_CATEGORY)#
positive_ins_sgpn = np.zeros(NUM_CATEGORY)
total_sgpn = np.zeros(NUM_CATEGORY)
at = 0.5
for shape_idx in range(len_pts_files):
cur_train_filename = test_file_list[shape_idx]
printout(flog, '%d / %d ...' % (shape_idx, len_pts_files))
printout(flog, 'Loading train file ' + cur_train_filename)
if not os.path.exists(cur_train_filename):
continue
cur_data, cur_group, _, cur_seg = provider.loadDataFile_with_groupseglabel_stanfordindoor(cur_train_filename)
seg_output = np.zeros_like(cur_seg)
segrefine_output = np.zeros_like(cur_seg)
group_output = np.zeros_like(cur_group)
pts_label_one_hot, pts_label_mask = model.convert_seg_to_one_hot(cur_seg)
pts_group_label, _ = model.convert_groupandcate_to_one_hot(cur_group)
num_data = cur_data.shape[0]
gap = 5e-3
volume_num = int(1. / gap)+1
volume = -1* np.ones([volume_num,volume_num,volume_num]).astype(np.int32)
volume_seg = -1* np.ones([volume_num,volume_num,volume_num]).astype(np.int32)
intersections = np.zeros(NUM_CATEGORY)
unions = np.zeros(NUM_CATEGORY)
for j in range(num_data):
print ("Processsing: Shape [%d] Block[%d]"%(shape_idx, j))
pts = cur_data[j,...]
feed_dict = {
pointclouds_ph: np.expand_dims(pts,0),
ptsseglabel_ph: np.expand_dims(pts_label_one_hot[j,...],0),
ptsgroup_label_ph: np.expand_dims(pts_group_label[j,...],0),
is_training_ph: is_training,
}
pts_corr_val0, pred_confidence_val0, ptsclassification_val0, pts_corr_label_val0 = \
sess.run([net_output['simmat'],
net_output['conf'],
net_output['semseg'],
group_mat_label],
feed_dict=feed_dict)
seg = cur_seg[j,...]
ins = cur_group[j,...]
pts_corr_val = np.squeeze(pts_corr_val0[0]) #NxG
pred_confidence_val = np.squeeze(pred_confidence_val0[0])
ptsclassification_val = np.argmax(np.squeeze(ptsclassification_val0[0]),axis=1)
seg = np.squeeze(seg)
# print label_bin
groupids_block, refineseg, group_seg = GroupMerging(pts_corr_val, pred_confidence_val, ptsclassification_val, label_bin) # yolo_to_groupt(pts_corr_val, pts_corr_label_val0[0], seg,t=5)
groupids = BlockMerging(volume, volume_seg, pts[:,6:], groupids_block.astype(np.int32), group_seg, gap)
seg_output[j,:] = ptsclassification_val
segrefine_output[j,:] = refineseg
group_output[j,:] = groupids
total_acc += float(np.sum(ptsclassification_val==seg))/ptsclassification_val.shape[0]
total_seen += 1
# if output_verbose:
# output_color_point_cloud(pts[:, 6:], groupids.astype(np.int32),
# os.path.join(OUTPUT_DIR, '%d_%d_grouppred.obj' % (shape_idx,j)))
###### Evaluation
### Instance Segmentation
## Pred
group_pred = group_output.reshape(-1)
seg_pred = seg_output.reshape(-1)
pts = cur_data.reshape([-1, 9])
# filtering
x = (pts[:, 6] / gap).astype(np.int32)
y = (pts[:, 7] / gap).astype(np.int32)
z = (pts[:, 8] / gap).astype(np.int32)
for i in range(group_pred.shape[0]):
if volume[x[i], y[i], z[i]] != -1:
group_pred[i] = volume[x[i], y[i], z[i]]
seg_gt = cur_seg.reshape(-1)
un = np.unique(group_pred)
pts_in_pred = [[] for itmp in range(NUM_CATEGORY)]
group_pred_final = -1 * np.ones_like(group_pred)
grouppred_cnt = 0
for ig, g in enumerate(un): #each object in prediction
if g == -1:
continue
tmp = (group_pred == g)
sem_seg_g = int(stats.mode(seg_pred[tmp])[0])
if np.sum(tmp) > 0.25 * min_num_pts_in_group[sem_seg_g]:
pts_in_pred[sem_seg_g] += [tmp]
group_pred_final[tmp] = grouppred_cnt
grouppred_cnt += 1
# savepts = pts[tmp, 6:]
# savegroup = group_pred[tmp]
# if output_verbose:
# output_color_point_cloud(savepts, savegroup.astype(np.int32),
# os.path.join(OUTPUT_DIR, '%d_%d_group.obj' % (sem_seg_g, ig)))
if output_verbose:
output_color_point_cloud(pts[:, 6:], seg_pred.astype(np.int32),
os.path.join(OUTPUT_DIR, '%d_segpred.obj' % (shape_idx)))
output_color_point_cloud(pts[:, 6:], group_pred_final.astype(np.int32),
os.path.join(OUTPUT_DIR, '%d_grouppred.obj' % (shape_idx)))
## GT
group_gt = cur_group.reshape(-1)
un = np.unique(group_gt)
pts_in_gt = [[] for itmp in range(NUM_CATEGORY)]
for ig, g in enumerate(un):
tmp = (group_gt == g)
sem_seg_g = int(stats.mode(seg_pred[tmp])[0])
pts_in_gt[sem_seg_g] += [tmp]
total_sgpn[sem_seg_g] += 1
for i_sem in range(NUM_CATEGORY):
tp = [0.] * len(pts_in_pred[i_sem])
fp = [0.] * len(pts_in_pred[i_sem])
gtflag = np.zeros(len(pts_in_gt[i_sem]))
for ip, ins_pred in enumerate(pts_in_pred[i_sem]):
ovmax = -1.
for ig, ins_gt in enumerate(pts_in_gt[i_sem]):
union = (ins_pred | ins_gt)
intersect = (ins_pred & ins_gt)
iou = float(np.sum(intersect)) / np.sum(union)
if iou > ovmax:
ovmax = iou
igmax = ig
if ovmax >= at:
if gtflag[igmax] == 0:
tp[ip] = 1 # true
gtflag[igmax] = 1
else:
fp[ip] = 1 # multiple det
else:
fp[ip] = 1 # false positive
tpsins[i_sem] += tp
fpsins[i_sem] += fp
### Semantic Segmentation
un, indices = np.unique(seg_gt, return_index=True)
for segid in un:
intersect = np.sum((seg_pred == segid) & (seg_gt == segid))
union = np.sum((seg_pred == segid) | (seg_gt == segid))
intersections[segid] += intersect
unions[segid] += union
iou = intersections / unions
for i_iou, iou_ in enumerate(iou):
if not np.isnan(iou_):
ious[i_iou] += iou_
totalnums[i_iou] += 1
ap = np.zeros(NUM_CATEGORY)
for i_sem in range(NUM_CATEGORY):
ap[i_sem], _, _ = eval_3d_perclass(tpsins[i_sem], fpsins[i_sem], total_sgpn[i_sem])
print 'Instance Segmentation AP:', ap
print 'Instance Segmentation mAP:', np.mean(ap)
print 'Semantic Segmentation IoU:', ious / totalnums
print 'Semantic Segmentation Acc: %f' , total_acc/total_seen