def estimate(area): LOG_DIR = 'log{}'.format(area) num_classes = 13 file_path = "data/train_hdf5_file_list_woArea{}.txt".format(area) train_file_list = provider.getDataFiles(file_path) mean_ins_size = np.zeros(num_classes) ptsnum_in_gt = [[] for itmp in range(num_classes)] train_data = [] train_group = [] train_sem = [] for h5_filename in train_file_list: cur_data, cur_group, _, cur_sem = provider.loadDataFile_with_groupseglabel_stanfordindoor(h5_filename) cur_data = np.reshape(cur_data, [-1, cur_data.shape[-1]]) cur_group = np.reshape(cur_group, [-1]) cur_sem = np.reshape(cur_sem, [-1]) un = np.unique(cur_group) for ig, g in enumerate(un): tmp = (cur_group == g) sem_seg_g = int(stats.mode(cur_sem[tmp])[0]) ptsnum_in_gt[sem_seg_g].append(np.sum(tmp)) for idx in range(num_classes): mean_ins_size[idx] = np.mean(ptsnum_in_gt[idx]).astype(np.int) print(mean_ins_size) np.savetxt(os.path.join(LOG_DIR, 'mean_ins_size.txt'),mean_ins_size)
def estimate(flags): num_classes = flags.num_cls if flags.dataset == 'S3DIS': train_file_list = [os.path.join(flags.data_root, line.strip()) for line in open(flags.input_list, 'r')] else: print("Error: Not support the dataset: ", flags.dataset) return mean_ins_size = np.zeros(num_classes) ptsnum_in_gt = [[] for itmp in range(num_classes)] for h5_filename in train_file_list: print(h5_filename) cur_data, cur_group, _, cur_sem = provider.loadDataFile_with_groupseglabel_stanfordindoor(h5_filename) cur_data = np.reshape(cur_data, [-1, cur_data.shape[-1]]) cur_group = np.reshape(cur_group, [-1]) cur_sem = np.reshape(cur_sem, [-1]) un = np.unique(cur_group) for ig, g in enumerate(un): tmp = (cur_group == g) sem_seg_g = int(stats.mode(cur_sem[tmp])[0]) ptsnum_in_gt[sem_seg_g].append(np.sum(tmp)) for idx in range(num_classes): mean_ins_size[idx] = np.mean(ptsnum_in_gt[idx]).astype(np.int) print(mean_ins_size) np.savetxt(os.path.join(flags.out_dir, 'mean_ins_size.txt'), mean_ins_size)
def estimate_mean_thresholds(): with tf.Graph().as_default(): with tf.device('/gpu:' + str(GPU_INDEX)): pointclouds_pl, labels_pl, sem_labels_pl = placeholder_inputs(BATCH_SIZE, NUM_POINT) is_training_pl = tf.placeholder(tf.bool, shape=()) # Get model pred_sem, pred_ins = get_model(pointclouds_pl, is_training_pl, NUM_CLASSES) loader = tf.train.Saver() # Create a session config = tf.ConfigProto() config.gpu_options.allow_growth = True config.allow_soft_placement = True config.log_device_placement = True sess = tf.Session(config=config) is_training = False # Restore all the variables loader.restore(sess, MODEL_PATH) print("Model restored") train_file_list = provider.getDataFiles(ESTIMATE_FILE_PATH) ths = np.zeros(NUM_CLASSES) cnt = np.zeros(NUM_CLASSES) ths_ = np.zeros(NUM_CLASSES) for shape_idx in range(len(train_file_list)): h5_filename = train_file_list[shape_idx] print('Loading file ' + h5_filename) cur_data, cur_group, _, cur_sem = provider.loadDataFile_with_groupseglabel_stanfordindoor(h5_filename) num_data = cur_data.shape[0] for j in range(num_data): print("Processing: Shape [%d] Block[%d]" % (shape_idx, j)) pts = cur_data[j, ...] seg = cur_sem[j, ...] ins = cur_group[j, ...] feed_dict = { pointclouds_pl: np.expand_dims(pts, 0), is_training_pl: is_training } pred_ins_val = sess.run([pred_ins], feed_dict=feed_dict) pred_ins_val = np.squeeze(pred_ins_val, axis=0) dis_mat = np.expand_dims(pred_ins_val, axis=1) - np.expand_dims(pred_ins_val, axis=0) dis_mat = np.linalg.norm(dis_mat, ord=1, axis=2) ths, ths_, cnt = Get_Ths(dis_mat, seg, ins, ths, ths_, cnt) print(cnt) ths = [ths[i] / cnt[i] if cnt[i] != 0 else 0.2 for i in range(len(cnt))] print(ths) np.savetxt(os.path.join(LOG_DIR, 'mean_thresholds.txt'), ths)
def load_data_file(input_file): cur_data, cur_group, _, cur_sem = provider.loadDataFile_with_groupseglabel_stanfordindoor( input_file) return cur_data, cur_sem, cur_group
def predict(): is_training = False with tf.device('/gpu:' + str(gpu_to_use)): is_training_ph = tf.placeholder(tf.bool, shape=()) pointclouds_ph, ptsseglabel_ph, ptsgroup_label_ph, _, _, _ = \ model.placeholder_inputs(BATCH_SIZE, POINT_NUM, NUM_GROUPS, NUM_CATEGORY) group_mat_label = tf.matmul( ptsgroup_label_ph, tf.transpose(ptsgroup_label_ph, perm=[0, 2, 1])) net_output = model.get_model(pointclouds_ph, is_training_ph, group_cate_num=NUM_CATEGORY) # Add ops to save and restore all the variables. saver = tf.train.Saver() config = tf.ConfigProto() config.gpu_options.allow_growth = True config.allow_soft_placement = True with tf.Session(config=config) as sess: # Restore variables from disk. ckptstate = tf.train.get_checkpoint_state(PRETRAINED_MODEL_PATH) if ckptstate is not None: LOAD_MODEL_FILE = os.path.join( PRETRAINED_MODEL_PATH, os.path.basename(ckptstate.model_checkpoint_path)) saver.restore(sess, LOAD_MODEL_FILE) print("Model loaded in file: %s" % LOAD_MODEL_FILE) else: print("Fail to load modelfile: %s" % PRETRAINED_MODEL_PATH) ths = np.zeros(NUM_CATEGORY) ths_ = np.zeros(NUM_CATEGORY) cnt = np.zeros(NUM_CATEGORY) min_groupsize = np.zeros(NUM_CATEGORY) min_groupsize_cnt = np.zeros(NUM_CATEGORY) for shape_idx in range(len_pts_files): cur_train_filename = test_file_list[shape_idx] if not os.path.exists(cur_train_filename): continue cur_data, cur_group, _, cur_seg = provider.loadDataFile_with_groupseglabel_stanfordindoor( cur_train_filename) if OUTPUT_VERBOSE: pts = np.reshape(cur_data, [-1, 9]) output_point_cloud_rgb( pts[:, 6:], pts[:, 3:6], os.path.join(OUTPUT_DIR, '%d_pts.obj' % (shape_idx))) pts_label_one_hot, pts_label_mask = model.convert_seg_to_one_hot( cur_seg) pts_group_label, _ = model.convert_groupandcate_to_one_hot( cur_group) num_data = cur_data.shape[0] cur_seg_flatten = np.reshape(cur_seg, [-1]) un, indices = np.unique(cur_group, return_index=True) for iu, u in enumerate(un): groupsize = np.sum(cur_group == u) groupcate = cur_seg_flatten[indices[iu]] min_groupsize[groupcate] += groupsize # print groupsize, min_groupsize[groupcate]/min_groupsize_cnt[groupcate] min_groupsize_cnt[groupcate] += 1 for j in range(num_data): print("Processsing: Shape [%d] Block[%d]" % (shape_idx, j)) pts = cur_data[j, ...] feed_dict = { pointclouds_ph: np.expand_dims(pts, 0), ptsseglabel_ph: np.expand_dims(pts_label_one_hot[j, ...], 0), ptsgroup_label_ph: np.expand_dims(pts_group_label[j, ...], 0), is_training_ph: is_training, } pts_corr_val0, pred_confidence_val0, ptsclassification_val0, pts_corr_label_val0 = \ sess.run([net_output['simmat'], net_output['conf'], net_output['semseg'], group_mat_label], feed_dict=feed_dict) seg = cur_seg[j, ...] ins = cur_group[j, ...] pts_corr_val = np.squeeze(pts_corr_val0[0]) pred_confidence_val = np.squeeze(pred_confidence_val0[0]) ptsclassification_val = np.argmax(np.squeeze( ptsclassification_val0[0]), axis=1) pts_corr_label_val = np.squeeze(1 - pts_corr_label_val0) seg = np.squeeze(seg) ins = np.squeeze(ins) ind = (seg == 8) pts_corr_val0 = (pts_corr_val > 1.).astype(np.float) print np.mean( np.transpose(np.abs(pts_corr_label_val[ind] - pts_corr_val0[ind]), axes=[1, 0])[ind]) ths, ths_, cnt = Get_Ths(pts_corr_val, seg, ins, ths, ths_, cnt) print ths / cnt if OUTPUT_VERBOSE: un, indices = np.unique(ins, return_index=True) for ii, id in enumerate(indices): corr = pts_corr_val[id].copy() output_scale_point_cloud( pts[:, 6:], np.float32(corr), os.path.join( OUTPUT_DIR, '%d_%d_%d_%d_scale.obj' % (shape_idx, j, un[ii], seg[id]))) corr = pts_corr_label_val[id] output_scale_point_cloud( pts[:, 6:], np.float32(corr), os.path.join( OUTPUT_DIR, '%d_%d_%d_%d_scalegt.obj' % (shape_idx, j, un[ii], seg[id]))) output_scale_point_cloud( pts[:, 6:], np.float32(pred_confidence_val), os.path.join(OUTPUT_DIR, '%d_%d_conf.obj' % (shape_idx, j))) output_color_point_cloud( pts[:, 6:], ptsclassification_val.astype(np.int32), os.path.join(OUTPUT_DIR, '%d_seg.obj' % (shape_idx))) ths = [ ths[i] / cnt[i] if cnt[i] != 0 else 0.2 for i in range(len(cnt)) ] np.savetxt(os.path.join(RESTORE_DIR, 'pergroup_thres.txt'), ths) min_groupsize = [ int(float(min_groupsize[i]) / min_groupsize_cnt[i]) if min_groupsize_cnt[i] != 0 else 0 for i in range(len(min_groupsize)) ] np.savetxt(os.path.join(RESTORE_DIR, 'mingroupsize.txt'), min_groupsize)
BN_INIT_DECAY = 0.5 BN_DECAY_DECAY_RATE = 0.5 #BN_DECAY_DECAY_STEP = float(DECAY_STEP * 2) BN_DECAY_DECAY_STEP = float(DECAY_STEP) BN_DECAY_CLIP = 0.99 HOSTNAME = socket.gethostname() # Load ALL data train_file_list = provider.getDataFiles(TRAINING_FILE_LIST) train_data = [] train_group = [] train_sem = [] for h5_filename in train_file_list: cur_data, cur_group, _, cur_sem = provider.loadDataFile_with_groupseglabel_stanfordindoor( h5_filename) train_data.append(cur_data) train_group.append(cur_group) train_sem.append(cur_sem) train_data = np.concatenate(train_data, axis=0) train_group = np.concatenate(train_group, axis=0) train_sem = np.concatenate(train_sem, axis=0) def log_string(out_str): LOG_FOUT.write(out_str + '\n') LOG_FOUT.flush() print(out_str) def get_learning_rate(batch):
def predict(): is_training = False with tf.device('/gpu:' + str(gpu_to_use)): is_training_ph = tf.placeholder(tf.bool, shape=()) pointclouds_ph, ptsseglabel_ph, ptsgroup_label_ph, _, _, _ = \ model.placeholder_inputs(BATCH_SIZE, POINT_NUM, NUM_GROUPS, NUM_CATEGORY) net_output = model.get_model(pointclouds_ph, is_training_ph, group_cate_num=NUM_CATEGORY) group_mat_label = tf.matmul(ptsgroup_label_ph, tf.transpose(ptsgroup_label_ph, perm=[0, 2, 1])) #BxNxN: (i,j) if i and j in the same group # Add ops to save and restore all the variables. saver = tf.train.Saver() config = tf.ConfigProto() config.gpu_options.allow_growth = True config.allow_soft_placement = True with tf.Session(config=config) as sess: ckptstate = tf.train.get_checkpoint_state(PRETRAINED_MODEL_PATH) if ckptstate is not None: LOAD_MODEL_FILE = os.path.join(PRETRAINED_MODEL_PATH,os.path.basename(ckptstate.model_checkpoint_path)) saver.restore(sess, LOAD_MODEL_FILE) print("Model loaded in file: %s" % LOAD_MODEL_FILE) else: print("Fail to load modelfile: %s" % PRETRAINED_MODEL_PATH) for shape_idx in range(len_pts_files): cur_train_filename = test_file_list[shape_idx] if not os.path.exists(cur_train_filename): continue cur_data, cur_group, _, cur_seg = provider.loadDataFile_with_groupseglabel_stanfordindoor(cur_train_filename) seg_output = np.zeros_like(cur_seg) segrefine_output = np.zeros_like(cur_seg) group_output = np.zeros_like(cur_group) conf_output = np.zeros_like(cur_group).astype(np.float) pts_label_one_hot, pts_label_mask = model.convert_seg_to_one_hot(cur_seg) pts_group_label, _ = model.convert_groupandcate_to_one_hot(cur_group) num_data = cur_data.shape[0] gap = 5e-3 volume_num = int(1. / gap)+1 volume = -1* np.ones([volume_num,volume_num,volume_num]).astype(np.int32) volume_seg = -1* np.ones([volume_num,volume_num,volume_num, NUM_CATEGORY]).astype(np.int32) intersections = np.zeros(NUM_CATEGORY) unions = np.zeros(NUM_CATEGORY) print('[%d / %d] Block Number: %d' % (shape_idx, len_pts_files, num_data)) print('Loading train file %s' % (cur_train_filename)) flag = True for j in range(num_data): pts = cur_data[j,...] feed_dict = { pointclouds_ph: np.expand_dims(pts,0), ptsseglabel_ph: np.expand_dims(pts_label_one_hot[j,...],0), ptsgroup_label_ph: np.expand_dims(pts_group_label[j,...],0), is_training_ph: is_training, } pts_corr_val0, pred_confidence_val0, ptsclassification_val0, pts_corr_label_val0 = \ sess.run([net_output['simmat'], net_output['conf'], net_output['semseg'], group_mat_label], feed_dict=feed_dict) seg = cur_seg[j,...] ins = cur_group[j,...] pts_corr_val = np.squeeze(pts_corr_val0[0]) #NxG pred_confidence_val = np.squeeze(pred_confidence_val0[0]) ptsclassification_val = np.argmax(np.squeeze(ptsclassification_val0[0]),axis=1) seg = np.squeeze(seg) # print label_bin try: groupids_block, refineseg, group_seg = GroupMerging_old(pts_corr_val, pred_confidence_val, ptsclassification_val, label_bin) # yolo_to_groupt(pts_corr_val, pts_corr_label_val0[0], seg,t=5) groupids = BlockMerging(volume, volume_seg, pts[:,6:], groupids_block.astype(np.int32), group_seg, gap) seg_output[j,:] = ptsclassification_val segrefine_output[j,:] = refineseg group_output[j,:] = groupids conf_output[j,:] = pred_confidence_val ###### Generate Results for Evaluation basefilename = os.path.basename(cur_train_filename).split('.')[-2] scene_fn = os.path.join(OUTPUT_DIR, '%s.txt' % basefilename) f_scene = open(scene_fn, 'w') scene_gt_fn = os.path.join(GT_DIR, '%s.txt' % basefilename) group_pred = group_output.reshape(-1) seg_pred = seg_output.reshape(-1) conf = conf_output.reshape(-1) pts = cur_data.reshape([-1, 9]) # filtering x = (pts[:, 6] / gap).astype(np.int32) y = (pts[:, 7] / gap).astype(np.int32) z = (pts[:, 8] / gap).astype(np.int32) for i in range(group_pred.shape[0]): if volume[x[i], y[i], z[i]] != -1: group_pred[i] = volume[x[i], y[i], z[i]] un = np.unique(group_pred) pts_in_pred = [[] for itmp in range(NUM_CATEGORY)] group_pred_final = -1 * np.ones_like(group_pred) grouppred_cnt = 0 for ig, g in enumerate(un): #each object in prediction if g == -1: continue obj_fn = "predicted_masks/%s_%d.txt" % (basefilename, ig) tmp = (group_pred == g) sem_seg_g = int(stats.mode(seg_pred[tmp])[0]) if np.sum(tmp) > 0.25 * min_num_pts_in_group[sem_seg_g]: pts_in_pred[sem_seg_g] += [tmp] group_pred_final[tmp] = grouppred_cnt conf_obj = np.mean(conf[tmp]) grouppred_cnt += 1 f_scene.write("%s %d %f\n" % (obj_fn, sem_seg_g, conf_obj)) np.savetxt(os.path.join(OUTPUT_DIR, obj_fn), tmp.astype(np.int), fmt='%d') seg_gt = cur_seg.reshape(-1) group_gt = cur_group.reshape(-1) groupid_gt = seg_gt * 1000 + group_gt np.savetxt(scene_gt_fn, groupid_gt.astype(np.int64), fmt='%d') f_scene.close() if output_verbose: output_color_point_cloud(pts[:, 6:], seg_pred.astype(np.int32), os.path.join(OUTPUT_DIR, '%s_segpred.obj' % (obj_fn))) output_color_point_cloud(pts[:, 6:], group_pred_final.astype(np.int32), os.path.join(OUTPUT_DIR, '%s_grouppred.obj' % (obj_fn)))
def test(): with tf.Graph().as_default(): with tf.device('/gpu:' + str(GPU_INDEX)): pointclouds_pl, labels_pl, sem_labels_pl = placeholder_inputs( BATCH_SIZE, NUM_POINT) is_training_pl = tf.placeholder(tf.bool, shape=()) # Get model pred_sem, pred_ins = get_model(pointclouds_pl, is_training_pl, NUM_CLASSES) pred_sem_softmax = tf.nn.softmax(pred_sem) pred_sem_label = tf.argmax(pred_sem_softmax, axis=2) loader = tf.train.Saver() # Create a session config = tf.ConfigProto() config.gpu_options.allow_growth = True config.allow_soft_placement = True config.log_device_placement = False sess = tf.Session(config=config) is_training = False # Restore variables from disk. loader.restore(sess, MODEL_PATH) logger.info("Model restored from {}".format(MODEL_PATH)) ops = { 'pointclouds_pl': pointclouds_pl, 'labels_pl': labels_pl, 'sem_labels_pl': sem_labels_pl, 'is_training_pl': is_training_pl, 'pred_ins': pred_ins, 'pred_sem_label': pred_sem_label, 'pred_sem_softmax': pred_sem_softmax } total_acc = 0.0 total_seen = 0 output_filelist_f = os.path.join(LOG_DIR, 'output_filelist.txt') fout_out_filelist = [] for shape_idx in range(len_pts_files): room_path = ROOM_PATH_LIST[shape_idx] out_data_label_filename = os.path.basename( room_path)[:-EXT_LEN] + '_pred.txt' out_data_label_filename = os.path.join(OUTPUT_DIR, out_data_label_filename) out_gt_label_filename = os.path.basename( room_path)[:-EXT_LEN] + '_gt.txt' out_gt_label_filename = os.path.join(OUTPUT_DIR, out_gt_label_filename) fout_data_label = [] fout_gt_label = [] fout_out_filelist.append(out_data_label_filename + '\n') logger.info('%d / %d ...' % (shape_idx, len_pts_files)) logger.info('Loading file ' + room_path) size_path = room_path if FILE_TYPE == 'hdf5': size_path = size_path.replace('indoor3d_ins_seg_hdf5', 'stanford_indoor3d_ins.sem') size_path = "{}.npy".format(size_path[:-3]) cur_data, cur_group, _, cur_sem = \ provider.loadDataFile_with_groupseglabel_stanfordindoor(room_path) elif FILE_TYPE == 'numpy': cur_data, cur_sem, cur_group = \ indoor3d_util.room2blocks_wrapper_normalized(room_path, NUM_POINT, block_size=1.0, stride=0.5, random_sample=False, sample_num=None) cur_data = cur_data[:, 0:NUM_POINT, :] cur_sem = np.squeeze(cur_sem) cur_group = np.squeeze(cur_group) # Get room dimension.. data_label = np.load(size_path) data = data_label[:, 0:6] max_room_x = max(data[:, 0]) max_room_y = max(data[:, 1]) max_room_z = max(data[:, 2]) cur_pred_sem = np.zeros_like(cur_sem) cur_pred_sem_softmax = np.zeros( [cur_sem.shape[0], cur_sem.shape[1], NUM_CLASSES]) group_output = np.zeros_like(cur_group) gap = 5e-3 volume_num = int(1. / gap) + 1 volume = -1 * np.ones([volume_num, volume_num, volume_num]).astype( np.int32) volume_seg = -1 * np.ones([volume_num, volume_num, volume_num ]).astype(np.int32) num_data = cur_data.shape[0] for j in range(num_data): logger.info("Processsing: Shape [%d] Block[%d]" % (shape_idx, j)) pts = cur_data[j, ...] group = cur_group[j] sem = cur_sem[j] feed_dict = { ops['pointclouds_pl']: np.expand_dims(pts, 0), ops['labels_pl']: np.expand_dims(group, 0), ops['sem_labels_pl']: np.expand_dims(sem, 0), ops['is_training_pl']: is_training } pred_ins_val, pred_sem_label_val, pred_sem_softmax_val = sess.run( [ ops['pred_ins'], ops['pred_sem_label'], ops['pred_sem_softmax'] ], feed_dict=feed_dict) pred_val = np.squeeze(pred_ins_val, axis=0) pred_sem = np.squeeze(pred_sem_label_val, axis=0) pred_sem_softmax = np.squeeze(pred_sem_softmax_val, axis=0) cur_pred_sem[j, :] = pred_sem cur_pred_sem_softmax[j, ...] = pred_sem_softmax # cluster group_seg = {} bandwidth = BANDWIDTH num_clusters, labels, cluster_centers = cluster( pred_val, bandwidth) for idx_cluster in range(num_clusters): tmp = (labels == idx_cluster) estimated_seg = int(stats.mode(pred_sem[tmp])[0]) group_seg[idx_cluster] = estimated_seg groupids_block = labels groupids = BlockMerging(volume, volume_seg, pts[:, 6:], groupids_block.astype(np.int32), group_seg, gap) group_output[j, :] = groupids total_acc += float(np.sum(pred_sem == sem)) / pred_sem.shape[0] total_seen += 1 group_pred = group_output.reshape(-1) seg_pred = cur_pred_sem.reshape(-1) seg_pred_softmax = cur_pred_sem_softmax.reshape([-1, NUM_CLASSES]) pts = cur_data.reshape([-1, 9]) # filtering x = (pts[:, 6] / gap).astype(np.int32) y = (pts[:, 7] / gap).astype(np.int32) z = (pts[:, 8] / gap).astype(np.int32) for i in range(group_pred.shape[0]): if volume[x[i], y[i], z[i]] != -1: group_pred[i] = volume[x[i], y[i], z[i]] seg_gt = cur_sem.reshape(-1) un = np.unique(group_pred) pts_in_pred = [[] for itmp in range(NUM_CLASSES)] group_pred_final = -1 * np.ones_like(group_pred) grouppred_cnt = 0 for ig, g in enumerate(un): # each object in prediction if g == -1: continue tmp = (group_pred == g) sem_seg_g = int(stats.mode(seg_pred[tmp])[0]) # if np.sum(tmp) > 500: if np.sum(tmp) > 0.25 * mean_num_pts_in_group[sem_seg_g]: group_pred_final[tmp] = grouppred_cnt pts_in_pred[sem_seg_g] += [tmp] grouppred_cnt += 1 pts[:, 6] *= max_room_x pts[:, 7] *= max_room_y pts[:, 8] *= max_room_z pts[:, 3:6] *= 255.0 ins = group_pred_final.astype(np.int32) sem = seg_pred.astype(np.int32) sem_softmax = seg_pred_softmax sem_gt = seg_gt ins_gt = cur_group.reshape(-1) for i in range(pts.shape[0]): fout_data_label.append( '%f %f %f %d %d %d %f %d %d\n' % (pts[i, 6], pts[i, 7], pts[i, 8], pts[i, 3], pts[i, 4], pts[i, 5], sem_softmax[i, sem[i]], sem[i], ins[i])) fout_gt_label.append('%d %d\n' % (sem_gt[i], ins_gt[i])) with open(out_data_label_filename, 'w') as fd: fd.writelines(fout_data_label) with open(out_gt_label_filename, 'w') as fd: fd.writelines(fout_gt_label) if output_verbose: # file name outfile_name = ROOM_PATH_LIST[shape_idx].split( '/')[-1][:-EXT_LEN] # Raw Point Cloud output_point_cloud_rgb( pts[:, 6:], pts[:, 3:6].astype(np.int32), os.path.join(VIS_DIR, '{}_raw.obj'.format(outfile_name))) logger.info('Saving file {}_raw.obj'.format(outfile_name)) # Instance Prediction output_color_point_cloud( pts[:, 6:], group_pred_final.astype(np.int32), os.path.join(VIS_DIR, '{}_pred_ins.obj'.format(outfile_name))) logger.info('Saving file {}_pred_ins.obj'.format(outfile_name)) # Semantic Prediction output_color_point_cloud( pts[:, 6:], seg_pred.astype(np.int32), os.path.join(VIS_DIR, '{}_pred_sem.obj'.format(outfile_name))) logger.info('Saving file {}_pred_sem.obj'.format(outfile_name)) # Instance Ground Truth output_color_point_cloud( pts[:, 6:], ins_gt, os.path.join(VIS_DIR, '{}_gt_ins.obj'.format(outfile_name))) logger.info('Saving file {}_gt_ins.obj'.format(outfile_name)) # Semantic Ground Truth output_color_point_cloud( pts[:, 6:], sem_gt, os.path.join(VIS_DIR, '{}_gt_sem.obj'.format(outfile_name))) logger.info('Saving file {}_gt_sem.obj'.format(outfile_name)) with open(output_filelist_f, 'w') as fd: fd.writelines(fout_out_filelist)
def train(): with tf.Graph().as_default(): with tf.device('/gpu:' + str(FLAGS.gpu)): batch = tf.Variable(0, trainable=False, name='batch') learning_rate = tf.train.exponential_decay( BASE_LEARNING_RATE, # base learning rate batch * BATCH_SIZE, # global_var indicating the number of steps DECAY_STEP, # step size DECAY_RATE, # decay rate staircase=True # Stair-case or continuous decreasing ) learning_rate = tf.maximum(learning_rate, LEARNING_RATE_CLIP) lr_op = tf.summary.scalar('learning_rate', learning_rate) pointclouds_ph, ptsseglabel_ph, ptsgroup_label_ph, pts_seglabel_mask_ph, pts_group_mask_ph, alpha_ph = \ model.placeholder_inputs(BATCH_SIZE, POINT_NUM, NUM_GROUPS, NUM_CATEGORY) is_training_ph = tf.placeholder(tf.bool, shape=()) labels = {'ptsgroup': ptsgroup_label_ph, 'semseg': ptsseglabel_ph, 'semseg_mask': pts_seglabel_mask_ph, 'group_mask': pts_group_mask_ph} net_output = model.get_model(pointclouds_ph, is_training_ph, group_cate_num=NUM_CATEGORY, m=MARGINS[0]) loss, grouperr, same, same_cnt, diff, diff_cnt, pos, pos_cnt = model.get_loss(net_output, labels, alpha_ph, MARGINS) total_training_loss_ph = tf.placeholder(tf.float32, shape=()) group_err_loss_ph = tf.placeholder(tf.float32, shape=()) total_train_loss_sum_op = tf.summary.scalar('total_training_loss', total_training_loss_ph) group_err_op = tf.summary.scalar('group_err_loss', group_err_loss_ph) train_variables = tf.trainable_variables() trainer = tf.train.AdamOptimizer(learning_rate) train_op = trainer.minimize(loss, var_list=train_variables, global_step=batch) loader = tf.train.Saver([v for v in tf.all_variables()#]) if ('conf_logits' not in v.name) and ('Fsim' not in v.name) and ('Fsconf' not in v.name) and ('batch' not in v.name) ]) saver = tf.train.Saver([v for v in tf.all_variables()]) config = tf.ConfigProto() config.gpu_options.allow_growth = True config.allow_soft_placement = True sess = tf.Session(config=config) init = tf.global_variables_initializer() sess.run(init) train_writer = tf.summary.FileWriter(SUMMARIES_FOLDER + '/train', sess.graph) import glob train_file_list= glob.glob(os.path.join(TRAINING_FILE_LIST,'*.h5')) # train_file_list=provider.getDataFiles(TRAINING_FILE_LIST) num_train_file = len(train_file_list) fcmd = open(os.path.join(LOG_STORAGE_PATH, 'cmd.txt'), 'w') fcmd.write(str(FLAGS)) fcmd.close() flog = open(os.path.join(LOG_STORAGE_PATH, 'log.txt'), 'w') ckptstate = tf.train.get_checkpoint_state(PRETRAINED_MODEL_PATH) if ckptstate is not None: LOAD_MODEL_FILE = os.path.join(PRETRAINED_MODEL_PATH, os.path.basename(ckptstate.model_checkpoint_path)) loader.restore(sess, LOAD_MODEL_FILE) printout(flog, "Model loaded in file: %s" % LOAD_MODEL_FILE) else: printout(flog, "Fail to load modelfile: %s" % PRETRAINED_MODEL_PATH) train_file_idx = np.arange(0, len(train_file_list)) np.random.shuffle(train_file_idx) ## load all data into memory all_data = [] all_group = [] all_seg = [] for i in range(num_train_file): cur_train_filename = train_file_list[train_file_idx[i]] # printout(flog, 'Loading train file ' + cur_train_filename) cur_data, cur_group, _, cur_seg = provider.loadDataFile_with_groupseglabel_stanfordindoor(cur_train_filename) all_data += [cur_data] all_group += [cur_group] all_seg += [cur_seg] all_data = np.concatenate(all_data,axis=0) all_group = np.concatenate(all_group,axis=0) all_seg = np.concatenate(all_seg,axis=0) num_data = all_data.shape[0] num_batch = num_data // BATCH_SIZE def train_one_epoch(epoch_num): ### NOTE: is_training = False: We do not update bn parameters during training due to the small batch size. This requires pre-training PointNet with large batchsize (say 32). is_training = False order = np.arange(num_data) np.random.shuffle(order) total_loss = 0.0 total_grouperr = 0.0 total_same = 0.0 total_diff = 0.0 total_pos = 0.0 same_cnt0 = 0 for j in range(num_batch): begidx = j * BATCH_SIZE endidx = (j + 1) * BATCH_SIZE pts_label_one_hot, pts_label_mask = model.convert_seg_to_one_hot(all_seg[order[begidx: endidx]]) pts_group_label, pts_group_mask = model.convert_groupandcate_to_one_hot(all_group[order[begidx: endidx]]) feed_dict = { pointclouds_ph: all_data[order[begidx: endidx], ...], ptsseglabel_ph: pts_label_one_hot, ptsgroup_label_ph: pts_group_label, pts_seglabel_mask_ph: pts_label_mask, pts_group_mask_ph: pts_group_mask, is_training_ph: is_training, alpha_ph: min(10., (float(epoch_num) / 5.) * 2. + 2.), } _, loss_val, simmat_val, grouperr_val, same_val, same_cnt_val, diff_val, diff_cnt_val, pos_val, pos_cnt_val = sess.run([train_op, loss, net_output['simmat'], grouperr, same, same_cnt, diff, diff_cnt, pos, pos_cnt], feed_dict=feed_dict) total_loss += loss_val total_grouperr += grouperr_val if diff_cnt_val!=0: total_diff += (diff_val / diff_cnt_val) if same_cnt_val > 0: total_same += same_val / same_cnt_val same_cnt0 += 1 total_pos += pos_val / pos_cnt_val if j % 10 == 9: if same_cnt0!=0: printout(flog, 'Batch: %d, loss: %f, grouperr: %f, same: %f, diff: %f, pos: %f' % (j, total_loss/10, total_grouperr/10, total_same/same_cnt0, total_diff/10, total_pos/10)) else: printout(flog, 'Batch: %d, loss: %f, grouperr: %f, same: %f, diff: %f, pos: %f' % (j, total_loss/10, total_grouperr/10, 1e10, total_diff/10, total_pos/10)) lr_sum, batch_sum, train_loss_sum, group_err_sum = sess.run( \ [lr_op, batch, total_train_loss_sum_op, group_err_op], \ feed_dict={total_training_loss_ph: total_loss / 10., group_err_loss_ph: total_grouperr / 10., }) train_writer.add_summary(train_loss_sum, batch_sum) train_writer.add_summary(lr_sum, batch_sum) train_writer.add_summary(group_err_sum, batch_sum) total_grouperr = 0.0 total_loss = 0.0 total_diff = 0.0 total_same = 0.0 total_pos = 0.0 same_cnt0 = 0 cp_filename = saver.save(sess, os.path.join(MODEL_STORAGE_PATH, 'epoch_' + str(epoch_num + 1) + '.ckpt')) printout(flog, 'Successfully store the checkpoint model into ' + cp_filename) if not os.path.exists(MODEL_STORAGE_PATH): os.mkdir(MODEL_STORAGE_PATH) for epoch in range(TRAINING_EPOCHES): printout(flog, '\n>>> Training for the epoch %d/%d ...' % (epoch, TRAINING_EPOCHES)) train_file_idx = np.arange(0, len(train_file_list)) np.random.shuffle(train_file_idx) train_one_epoch(epoch) flog.flush() cp_filename = saver.save(sess, os.path.join(MODEL_STORAGE_PATH, 'epoch_' + str(epoch + 1) + '.ckpt')) printout(flog, 'Successfully store the checkpoint model into ' + cp_filename) flog.close()
def predict(): is_training = False with tf.device('/gpu:' + str(gpu_to_use)): is_training_ph = tf.placeholder(tf.bool, shape=()) pointclouds_ph, ptsseglabel_ph, ptsgroup_label_ph, _, _, _ = \ model.placeholder_inputs(BATCH_SIZE, POINT_NUM, NUM_GROUPS, NUM_CATEGORY) net_output = model.get_model(pointclouds_ph, is_training_ph, group_cate_num=NUM_CATEGORY) group_mat_label = tf.matmul(ptsgroup_label_ph, tf.transpose(ptsgroup_label_ph, perm=[0, 2, 1])) #BxNxN: (i,j) if i and j in the same group # Add ops to save and restore all the variables. saver = tf.train.Saver() config = tf.ConfigProto() config.gpu_options.allow_growth = True config.allow_soft_placement = True with tf.Session(config=config) as sess: flog = open(os.path.join(OUTPUT_DIR, 'log.txt'), 'w') # Restore variables from disk. ckptstate = tf.train.get_checkpoint_state(PRETRAINED_MODEL_PATH) if ckptstate is not None: LOAD_MODEL_FILE = os.path.join(PRETRAINED_MODEL_PATH,os.path.basename(ckptstate.model_checkpoint_path)) saver.restore(sess, LOAD_MODEL_FILE) printout(flog, "Model loaded in file: %s" % LOAD_MODEL_FILE) else: printout(flog, "Fail to load modelfile: %s" % PRETRAINED_MODEL_PATH) total_acc = 0.0 total_seen = 0 ious = np.zeros(NUM_CATEGORY) totalnums = np.zeros(NUM_CATEGORY) tpsins = [[] for itmp in range(NUM_CATEGORY)]#= np.array([]).reshape(0, NUM_CATEGORY)#np.zeros(NUM_CATEGORY)# fpsins = [[] for itmp in range(NUM_CATEGORY)]#= np.array([]).reshape(0, NUM_CATEGORY)#np.zeros(NUM_CATEGORY)# positive_ins_sgpn = np.zeros(NUM_CATEGORY) total_sgpn = np.zeros(NUM_CATEGORY) at = 0.5 for shape_idx in range(len_pts_files): cur_train_filename = test_file_list[shape_idx] printout(flog, '%d / %d ...' % (shape_idx, len_pts_files)) printout(flog, 'Loading train file ' + cur_train_filename) if not os.path.exists(cur_train_filename): continue cur_data, cur_group, _, cur_seg = provider.loadDataFile_with_groupseglabel_stanfordindoor(cur_train_filename) seg_output = np.zeros_like(cur_seg) segrefine_output = np.zeros_like(cur_seg) group_output = np.zeros_like(cur_group) pts_label_one_hot, pts_label_mask = model.convert_seg_to_one_hot(cur_seg) pts_group_label, _ = model.convert_groupandcate_to_one_hot(cur_group) num_data = cur_data.shape[0] gap = 5e-3 volume_num = int(1. / gap)+1 volume = -1* np.ones([volume_num,volume_num,volume_num]).astype(np.int32) volume_seg = -1* np.ones([volume_num,volume_num,volume_num]).astype(np.int32) intersections = np.zeros(NUM_CATEGORY) unions = np.zeros(NUM_CATEGORY) for j in range(num_data): print ("Processsing: Shape [%d] Block[%d]"%(shape_idx, j)) pts = cur_data[j,...] feed_dict = { pointclouds_ph: np.expand_dims(pts,0), ptsseglabel_ph: np.expand_dims(pts_label_one_hot[j,...],0), ptsgroup_label_ph: np.expand_dims(pts_group_label[j,...],0), is_training_ph: is_training, } pts_corr_val0, pred_confidence_val0, ptsclassification_val0, pts_corr_label_val0 = \ sess.run([net_output['simmat'], net_output['conf'], net_output['semseg'], group_mat_label], feed_dict=feed_dict) seg = cur_seg[j,...] ins = cur_group[j,...] pts_corr_val = np.squeeze(pts_corr_val0[0]) #NxG pred_confidence_val = np.squeeze(pred_confidence_val0[0]) ptsclassification_val = np.argmax(np.squeeze(ptsclassification_val0[0]),axis=1) seg = np.squeeze(seg) # print label_bin groupids_block, refineseg, group_seg = GroupMerging(pts_corr_val, pred_confidence_val, ptsclassification_val, label_bin) # yolo_to_groupt(pts_corr_val, pts_corr_label_val0[0], seg,t=5) groupids = BlockMerging(volume, volume_seg, pts[:,6:], groupids_block.astype(np.int32), group_seg, gap) seg_output[j,:] = ptsclassification_val segrefine_output[j,:] = refineseg group_output[j,:] = groupids total_acc += float(np.sum(ptsclassification_val==seg))/ptsclassification_val.shape[0] total_seen += 1 # if output_verbose: # output_color_point_cloud(pts[:, 6:], groupids.astype(np.int32), # os.path.join(OUTPUT_DIR, '%d_%d_grouppred.obj' % (shape_idx,j))) ###### Evaluation ### Instance Segmentation ## Pred group_pred = group_output.reshape(-1) seg_pred = seg_output.reshape(-1) pts = cur_data.reshape([-1, 9]) # filtering x = (pts[:, 6] / gap).astype(np.int32) y = (pts[:, 7] / gap).astype(np.int32) z = (pts[:, 8] / gap).astype(np.int32) for i in range(group_pred.shape[0]): if volume[x[i], y[i], z[i]] != -1: group_pred[i] = volume[x[i], y[i], z[i]] seg_gt = cur_seg.reshape(-1) un = np.unique(group_pred) pts_in_pred = [[] for itmp in range(NUM_CATEGORY)] group_pred_final = -1 * np.ones_like(group_pred) grouppred_cnt = 0 for ig, g in enumerate(un): #each object in prediction if g == -1: continue tmp = (group_pred == g) sem_seg_g = int(stats.mode(seg_pred[tmp])[0]) if np.sum(tmp) > 0.25 * min_num_pts_in_group[sem_seg_g]: pts_in_pred[sem_seg_g] += [tmp] group_pred_final[tmp] = grouppred_cnt grouppred_cnt += 1 # savepts = pts[tmp, 6:] # savegroup = group_pred[tmp] # if output_verbose: # output_color_point_cloud(savepts, savegroup.astype(np.int32), # os.path.join(OUTPUT_DIR, '%d_%d_group.obj' % (sem_seg_g, ig))) if output_verbose: output_color_point_cloud(pts[:, 6:], seg_pred.astype(np.int32), os.path.join(OUTPUT_DIR, '%d_segpred.obj' % (shape_idx))) output_color_point_cloud(pts[:, 6:], group_pred_final.astype(np.int32), os.path.join(OUTPUT_DIR, '%d_grouppred.obj' % (shape_idx))) ## GT group_gt = cur_group.reshape(-1) un = np.unique(group_gt) pts_in_gt = [[] for itmp in range(NUM_CATEGORY)] for ig, g in enumerate(un): tmp = (group_gt == g) sem_seg_g = int(stats.mode(seg_pred[tmp])[0]) pts_in_gt[sem_seg_g] += [tmp] total_sgpn[sem_seg_g] += 1 for i_sem in range(NUM_CATEGORY): tp = [0.] * len(pts_in_pred[i_sem]) fp = [0.] * len(pts_in_pred[i_sem]) gtflag = np.zeros(len(pts_in_gt[i_sem])) for ip, ins_pred in enumerate(pts_in_pred[i_sem]): ovmax = -1. for ig, ins_gt in enumerate(pts_in_gt[i_sem]): union = (ins_pred | ins_gt) intersect = (ins_pred & ins_gt) iou = float(np.sum(intersect)) / np.sum(union) if iou > ovmax: ovmax = iou igmax = ig if ovmax >= at: if gtflag[igmax] == 0: tp[ip] = 1 # true gtflag[igmax] = 1 else: fp[ip] = 1 # multiple det else: fp[ip] = 1 # false positive tpsins[i_sem] += tp fpsins[i_sem] += fp ### Semantic Segmentation un, indices = np.unique(seg_gt, return_index=True) for segid in un: intersect = np.sum((seg_pred == segid) & (seg_gt == segid)) union = np.sum((seg_pred == segid) | (seg_gt == segid)) intersections[segid] += intersect unions[segid] += union iou = intersections / unions for i_iou, iou_ in enumerate(iou): if not np.isnan(iou_): ious[i_iou] += iou_ totalnums[i_iou] += 1 ap = np.zeros(NUM_CATEGORY) for i_sem in range(NUM_CATEGORY): ap[i_sem], _, _ = eval_3d_perclass(tpsins[i_sem], fpsins[i_sem], total_sgpn[i_sem]) print 'Instance Segmentation AP:', ap print 'Instance Segmentation mAP:', np.mean(ap) print 'Semantic Segmentation IoU:', ious / totalnums print 'Semantic Segmentation Acc: %f' , total_acc/total_seen