def main(): for LOG_DIR in FLAGS.log_dir: CKPT_DIR = os.path.join(LOG_DIR, 'trained_models') if not os.path.exists(LOG_DIR): print('ERROR: log_dir %s does not exist! Please Check!' % LOG_DIR) exit(1) VALID_DIR = os.path.join(LOG_DIR, FLAGS.valid_dir) if not os.path.exists(VALID_DIR): print('ERROR: valid_dir %s does not exist! Run valid.py first!' % VALID_DIR) exit(1) LOG_DIR = os.path.join(LOG_DIR, FLAGS.eval_dir) check_mkdir(LOG_DIR) PRED_DIR = os.path.join(LOG_DIR, FLAGS.pred_dir) force_mkdir(PRED_DIR) if FLAGS.visu_dir is not None: VISU_DIR = os.path.join(LOG_DIR, FLAGS.visu_dir) force_mkdir(VISU_DIR) os.system('cp %s %s' % (MODEL_FILE, LOG_DIR)) # bkp of model def os.system('cp %s %s' % (__file__, LOG_DIR)) # bkp of train procedure LOG_FOUT = open(os.path.join(LOG_DIR, 'log_eval.txt'), 'w') LOG_FOUT.write(str(FLAGS)+'\n') data_in_dir = '../../data/ins_seg_h5_for_sgpn/%s-%d/' % (FLAGS.category, FLAGS.level_id) test_h5_fn_list = [] for item in os.listdir(data_in_dir): if item.endswith('.h5') and item.startswith('test-'): test_h5_fn_list.append(os.path.join(data_in_dir, item)) NUM_CLASSES = 1 print('force Semantic Labels: ', NUM_CLASSES) NUM_INS = FLAGS.num_ins print('Number of Instances: ', NUM_INS) # load validiation hyper-parameters pw_sim_thres = np.loadtxt(os.path.join(VALID_DIR, 'per_category_pointwise_similarity_threshold.txt')).reshape(NUM_CLASSES) avg_group_size = np.loadtxt(os.path.join(VALID_DIR, 'per_category_average_group_size.txt')).reshape(NUM_CLASSES) min_group_size = 0.25 * avg_group_size for i in range(NUM_CLASSES): print('%d %f %d' % (i, pw_sim_thres[i], min_group_size[i])) # main log_string('pid: %s'%(str(os.getpid())), LOG_FOUT) eval(NUM_CLASSES=NUM_CLASSES, NUM_POINT=NUM_POINT, NUM_INS=NUM_INS, CKPT_DIR=CKPT_DIR, test_h5_fn_list=test_h5_fn_list, min_group_size=min_group_size, pw_sim_thres=pw_sim_thres, PRED_DIR=PRED_DIR, VISU_DIR=VISU_DIR, LOG_FOUT=LOG_FOUT) LOG_FOUT.close()
from progressbar import ProgressBar from subprocess import call from PIL import Image import scipy.misc as misc from detect_adj import compute_adj from detect_ref_sym import compute_ref_sym, atob_ref_sym from detect_trans_sym import compute_trans_sym, atob_trans_sym from detect_rot_sym import compute_rot_sym, atob_rot_sym anno_id = sys.argv[1] in_dir = os.path.join('../../data', anno_id) render_dir = os.path.join(in_dir, 'parts_render_after_merging') out_dir = os.path.join('results', anno_id) check_mkdir(out_dir) visu_dir = os.path.join(out_dir, 'visu') force_mkdir(visu_dir) parent_dir = os.path.join(visu_dir, 'parent') os.mkdir(parent_dir) info_dir = os.path.join(visu_dir, 'info') os.mkdir(info_dir) child_dir = os.path.join(visu_dir, 'child') os.mkdir(child_dir) json_fn = os.path.join(in_dir, 'result_after_merging.json') with open(json_fn, 'r') as fin: data = json.load(fin)[0] found_edges = dict()
def eval_per_class_ap(stat_fn, gt_dir, pred_dir, iou_threshold=0.5, plot_dir=None): """ Input: stat_fn contains all part ids and names gt_dir contains test-xx.h5 pred_dir contains test-xx.h5 Output: aps: Average Prediction Scores for each part category, evaluated on all test shapes mAP: mean AP """ print('Evaluation Start.') print('Ground-truth Directory: %s' % gt_dir) print('Prediction Directory: %s' % pred_dir) if plot_dir is not None: check_mkdir(plot_dir) # read stat_fn with open(stat_fn, 'r') as fin: part_name_list = [item.rstrip().split()[1] for item in fin.readlines()] print('Part Name List: ', part_name_list) n_labels = len(part_name_list) print('Total Number of Semantic Labels: %d' % n_labels) # check all h5 files test_h5_list = [] for item in os.listdir(gt_dir): if item.startswith('test-') and item.endswith('.h5'): if not os.path.exists(os.path.join(pred_dir, item)): print('ERROR: h5 file %s is in gt directory but not in pred directory.') exit(1) test_h5_list.append(item) # read each h5 file and collect per-part-category true_pos, false_pos and confidence scores true_pos_list = [[] for item in part_name_list] false_pos_list = [[] for item in part_name_list] conf_score_list = [[] for item in part_name_list] gt_npos = np.zeros((n_labels), dtype=np.int32) for item in test_h5_list: print('Testing %s' % item) gt_mask, gt_mask_label, gt_mask_valid, gt_mask_other = load_gt_h5(os.path.join(gt_dir, item)) pred_mask, pred_label, pred_valid, pred_conf = load_pred_h5(os.path.join(pred_dir, item)) n_shape = gt_mask.shape[0] gt_n_ins = gt_mask.shape[1] pred_n_ins = pred_mask.shape[1] for i in range(n_shape): cur_pred_mask = pred_mask[i, ...] cur_pred_label = pred_label[i, :] cur_pred_conf = pred_conf[i, :] cur_pred_valid = pred_valid[i, :] cur_gt_mask = gt_mask[i, ...] cur_gt_label = gt_mask_label[i, :] cur_gt_valid = gt_mask_valid[i, :] cur_gt_other = gt_mask_other[i, :] # classify all valid gt masks by part categories gt_mask_per_cat = [[] for item in part_name_list] for j in range(gt_n_ins): if cur_gt_valid[j]: sem_id = cur_gt_label[j] gt_mask_per_cat[sem_id].append(j) gt_npos[sem_id] += 1 # sort prediction and match iou to gt masks cur_pred_conf[~cur_pred_valid] = 0.0 order = np.argsort(-cur_pred_conf) gt_used = np.zeros((gt_n_ins), dtype=np.bool) for j in range(pred_n_ins): idx = order[j] if cur_pred_valid[idx]: sem_id = cur_pred_label[idx] iou_max = 0.0; cor_gt_id = -1; for k in gt_mask_per_cat[sem_id]: if not gt_used[k]: # Remove points with gt label *other* from the prediction # We will not evaluate them in the IoU since they can be assigned any label clean_cur_pred_mask = (cur_pred_mask[idx, :] & (~cur_gt_other)) intersect = np.sum(cur_gt_mask[k, :] & clean_cur_pred_mask) union = np.sum(cur_gt_mask[k, :] | clean_cur_pred_mask) iou = intersect * 1.0 / union if iou > iou_max: iou_max = iou cor_gt_id = k if iou_max > iou_threshold: gt_used[cor_gt_id] = True # add in a true positive true_pos_list[sem_id].append(True) false_pos_list[sem_id].append(False) conf_score_list[sem_id].append(cur_pred_conf[idx]) else: # add in a false positive true_pos_list[sem_id].append(False) false_pos_list[sem_id].append(True) conf_score_list[sem_id].append(cur_pred_conf[idx]) # compute per-part-category AP aps = np.zeros((n_labels), dtype=np.float32) ap_valids = np.ones((n_labels), dtype=np.bool) for i in range(n_labels): has_pred = (len(true_pos_list[i]) > 0) has_gt = (gt_npos[i] > 0) if not has_gt: ap_valids[i] = False continue if has_gt and not has_pred: continue cur_true_pos = np.array(true_pos_list[i], dtype=np.float32) cur_false_pos = np.array(false_pos_list[i], dtype=np.float32) cur_conf_score = np.array(conf_score_list[i], dtype=np.float32) # sort according to confidence score again order = np.argsort(-cur_conf_score) sorted_true_pos = cur_true_pos[order] sorted_false_pos = cur_false_pos[order] out_plot_fn = None if plot_dir is not None: out_plot_fn = os.path.join(plot_dir, part_name_list[i].replace('/', '-')+'.png') aps[i] = compute_ap(sorted_true_pos, sorted_false_pos, gt_npos[i], plot_fn=out_plot_fn) # compute mean AP mean_ap = np.sum(aps * ap_valids) / np.sum(ap_valids) return aps, ap_valids, gt_npos, mean_ap
def eval_recall_iou_nosem_ab(stat_fn, gt_dir, pred_dir, level_id, iou_threshold=0.5, plot_dir=None): """ Input: stat_fn contains all part ids and names gt_dir contains test-xx.h5 pred_dir contains test-xx.h5 Output: aps: Average Prediction Scores for each part category, evaluated on all test shapes mAP: mean AP """ #print('Evaluation Start.') #print('Ground-truth Directory: %s' % gt_dir) #print('Prediction Directory: %s' % pred_dir) if plot_dir is not None and not os.path.exists(plot_dir): check_mkdir(plot_dir) # read stat_fn with open(stat_fn, 'r') as fin: part_name_list = [item.rstrip().split()[1] for item in fin.readlines()] #print('Part Name List: ', part_name_list) n_labels = len(part_name_list) #print('Total Number of Semantic Labels: %d' % n_labels) # check all h5 files test_h5_list = [] for item in os.listdir(gt_dir): if item.startswith('test-') and item.endswith('.h5'): if not os.path.exists(os.path.join(pred_dir, 'Level_3', item)): print( 'ERROR: h5 file %s is in gt directory but not in pred directory.' ) #exit(1) test_h5_list.append(item) # read each h5 file and collect per-part-category true_pos, false_pos and confidence scores #true_pos_list = [[] for item in part_name_list] #false_pos_list = [[] for item in part_name_list] #conf_score_list = [[] for item in part_name_list] true_pos_list = [] false_pos_list = [] conf_score_list = [] iou_list = [] gt_npos = np.zeros((n_labels), dtype=np.int32) gt_ins_num = 0 for item in test_h5_list: print('Testing %s' % item) gt_mask, gt_mask_label, gt_mask_valid, gt_mask_other = load_gt_h5( os.path.join(gt_dir, item)) pred_mask, pred_valid, pred_conf = load_pred_h5_nosem( os.path.join(pred_dir, 'Level_3', item)) n_shape = gt_mask.shape[0] gt_n_ins = gt_mask.shape[1] pred_n_ins = pred_mask.shape[1] for i in range(n_shape): cur_pred_mask = pred_mask[i, ...] cur_pred_conf = pred_conf[i, :] cur_pred_valid = pred_valid[i, :] cur_gt_mask = gt_mask[i, ...] cur_gt_valid = gt_mask_valid[i, :] cur_gt_other = gt_mask_other[i, :] gt_ins_num += np.sum(cur_gt_valid) ## classify all valid gt masks by part categories #gt_mask_per_cat = [[] for item in part_name_list] #for j in range(gt_n_ins): # if cur_gt_valid[j]: # sem_id = cur_gt_label[j] # gt_mask_per_cat[sem_id].append(j) # gt_npos[sem_id] += 1 # sort prediction and match iou to gt masks assert np.sum(cur_gt_mask[np.sum(cur_gt_valid):]) == 0 cur_gt_mask = cur_gt_mask[:np.sum(cur_gt_valid)] cur_pred_mask = cur_pred_mask[np.where(cur_pred_valid)] # remove other mask cur_pred_mask = cur_pred_mask & ~np.tile( np.expand_dims(cur_gt_other, axis=0), [cur_pred_mask.shape[0], 1]) intersect = np.matmul(cur_gt_mask.astype( np.int32), (cur_pred_mask.transpose(1, 0).astype(np.int32))) union = cur_gt_mask.shape[1] - np.matmul( 1 - cur_gt_mask.astype(np.int32), 1 - (cur_pred_mask.transpose(1, 0).astype(np.int32))) iou = intersect / (union + 1e-6) row, column = linear_sum_assignment(-iou) iou_list += list(iou[row, column]) if row.shape[0] < cur_gt_mask.shape[0]: iou_list += list(np.zeros(cur_gt_mask.shape[0] - row.shape[0])) iou_arr = np.array(iou_list) recall_iou_list = [] #xx = np.linspace(0.05,0.55,11)-0.05+0.5 xx = np.linspace(0.05, 0.5, 10) - 0.05 + 0.5 for i in range(10): p = xx[i] recall_iou_list.append(np.sum(iou_arr >= p) / iou_arr.shape[0]) recall_iou_arr = np.array(recall_iou_list) if plot_dir is not None: plot_fn = os.path.join(plot_dir, gt_dir.split('/')[-2] + '_realliou' + '.png') import matplotlib.pyplot as plt import matplotlib matplotlib.use('Agg') fig = plt.figure() plt.plot(xx, recall_iou_arr, 'b-') plt.title('Recall versus IoU (AR: %4.2f%%)' % (np.mean(recall_iou_arr))) plt.xlabel('IoU') plt.ylabel('Recall') plt.xlim([0.5, 1]) plt.ylim([0, 1]) fig.savefig(plot_fn) plt.close(fig) # compute mean AP #mean_ap = np.sum(aps * ap_valids) / np.sum(ap_valids) return recall_iou_arr #, ap_valids, gt_npos, mean_ap
# Input&Output Settings parser.add_argument( '--log_dir', type=str, default='log', help= 'Directory that stores all training logs and trained models [default: log]' ) FLAGS = parser.parse_args() # save settings MODEL = importlib.import_module(FLAGS.model) # import network module MODEL_FILE = os.path.join(BASE_DIR, 'models', FLAGS.model + '.py') LOG_DIR = FLAGS.log_dir check_mkdir(LOG_DIR) os.system('cp %s %s' % (MODEL_FILE, LOG_DIR)) # bkp of model def os.system('cp %s %s' % (__file__, LOG_DIR)) # bkp of train procedure flog = open(os.path.join(LOG_DIR, 'log_train.txt'), 'w') flog.write(str(FLAGS) + '\n') def printout(flog, data): print(data) flog.write(data + '\n') # load meta data files stat_in_fn = '../../stats/after_merging_label_ids/%s-level-%d.txt' % ( FLAGS.category, FLAGS.level_id)