print('Target count: {:d}'.format(tars)) print('TP count: {:d}'.format(tps)) print('Total recall: {:.4f}'.format(total_recall)) print('Mean object recall: {:.4f}'.format(mean_obj_recall)) print('Mean scene recall: {:.4f}'.format(mean_scene_recall)) # print('Object recalls: {}'.format(str(obj_recalls))) # print('Scene recalls: {}'.format(str(scene_recalls))) print('') # Evaluation signature eval_sign = '' if error_type in ['add', 'adi']: eval_sign = 'thf=' + str(error_thresh_fact[error_type]) else: eval_sign = 'th=' + str(error_thresh[error_type]) # Save scores print('Saving scores...') scores_path = scores_mpath.format(error_path=error_path, eval_sign=eval_sign) inout.save_yaml(scores_path, scores) # Save matches print('Saving matches...') matches_path = matches_mpath.format(error_path=error_path, eval_sign=eval_sign) inout.save_yaml(matches_path, matches) print('') print('Done.')
# provided for the SIXD Challenge 2017. rgb = cv2.resize(rgb, par['cam']['im_size'], interpolation=cv2.INTER_AREA) #rgb = scipy.misc.imresize(rgb, par['cam']['im_size'][::-1], 'bicubic') # Save the rendered images inout.save_im(out_rgb_mpath.format(obj_id, im_id), rgb) inout.save_depth(out_depth_mpath.format(obj_id, im_id), depth) # Get 2D bounding box of the object model at the ground truth pose ys, xs = np.nonzero(depth > 0) obj_bb = misc.calc_2d_bbox(xs, ys, par['cam']['im_size']) obj_info[im_id] = { 'cam_K': par['cam']['K'].flatten().tolist(), 'view_level': int(views_level[view_id]), #'sphere_radius': float(radius) } obj_gt[im_id] = [{ 'cam_R_m2c': view['R'].flatten().tolist(), 'cam_t_m2c': view['t'].flatten().tolist(), 'obj_bb': [int(x) for x in obj_bb], 'obj_id': int(obj_id) }] im_id += 1 # Save metadata inout.save_yaml(out_obj_info_path.format(obj_id), obj_info) inout.save_yaml(out_obj_gt_path.format(obj_id), obj_gt)
vis = 0.5 * depth_im_vis + 0.5 * visib_gt_vis vis[vis > 1] = 1 vis_path = vis_mpath.format(dataset, delta, data_id, im_id, gt_id) inout.save_im(vis_path, vis) # Mask of depth differences below delta # mask_below_delta_vis = np.dstack([mask_below_delta, # zero_ch, zero_ch]) # vis_delta = 0.5 * depth_im_vis + 0.5 * mask_below_delta_vis # vis_delta[vis_delta > 1] = 1 # vis_delta_path = vis_delta_mpath.format( # dataset, delta, data_id, im_id, gt_id, delta) # inout.save_im(vis_delta_path, vis_delta) res_path = dp[gt_stats_mpath_key].format(data_id, delta) misc.ensure_dir(os.path.dirname(res_path)) inout.save_yaml(res_path, gt_stats) # visib_to_below_delta_fracs = sorted(visib_to_below_delta_fracs, # key=lambda x: x['frac'], reverse=True) # for i in range(200): # e = visib_to_below_delta_fracs[i] # print('{}: data_id: {}, im_id: {}, gt_id: {}, frac: {}'.format( # i, e['data_id'], e['im_id'], e['gt_id'], e['frac'] # )) # # import matplotlib.pyplot as plt # plt.plot([e['frac'] for e in visib_to_below_delta_fracs]) # plt.show()
def main(): # Paths to pose errors (calculated using eval_calc_errors.py) #--------------------------------------------------------------------------- top_level_path = os.path.dirname(os.path.dirname( os.path.abspath(__file__))) dataset = 'hinterstoisser' # dataset = 'tless' # dataset = 'tudlight' # dataset = 'rutgers' # dataset = 'tejani' # dataset = 'doumanoglou' # dataset = 'toyotalight' error_bpath = pjoin(top_level_path, 'eval') error_paths = [ pjoin(error_bpath, 'patch-linemod_' + dataset), # pjoin(error_bpath, 'hodan-iros15_tless_primesense'), ] error_dir = 'error=vsd_ntop=1_delta=15_tau=20_cost=step' for i in range(len(error_paths)): error_paths[i] = os.path.join(error_paths[i], error_dir) # Other paths #--------------------------------------------------------------------------- # Mask of path to the input file with calculated errors errors_mpath = pjoin('{error_path}', 'errors_{scene_id:02d}.yml') # Mask of path to the output file with established matches and calculated scores matches_mpath = pjoin('{error_path}', 'matches_{eval_sign}.yml') scores_mpath = pjoin('{error_path}', 'scores_{eval_sign}.yml') # Parameters #--------------------------------------------------------------------------- use_image_subset = True # Whether to use the specified subset of images require_all_errors = True # Whether to break if some errors are missing visib_gt_min = 0.1 # Minimum visible surface fraction of valid GT pose visib_delta = 15 # [mm] # Threshold of correctness error_thresh = { 'vsd': 0.3, 'cou': 0.5, 'te': 5.0, # [cm] 're': 5.0 # [deg] } # Factor k; threshold of correctness = k * d, where d is the object diameter error_thresh_fact = {'add': 0.1, 'adi': 0.1} # Evaluation #--------------------------------------------------------------------------- for error_path in error_paths: # Parse info about the errors from the folder names error_sign = os.path.basename(error_path) error_type = error_sign.split('_')[0].split('=')[1] n_top = int(error_sign.split('_')[1].split('=')[1]) res_sign = os.path.basename(os.path.dirname(error_path)).split('_') method = res_sign[0] dataset = res_sign[1] test_type = res_sign[2] if len(res_sign) > 3 else '' # Evaluation signature if error_type in ['add', 'adi']: eval_sign = 'thf=' + str(error_thresh_fact[error_type]) else: eval_sign = 'th=' + str(error_thresh[error_type]) eval_sign += '_min-visib=' + str(visib_gt_min) print('--- Processing: {}, {}, {}'.format(method, dataset, error_type)) # Load dataset parameters dp = get_dataset_params(dataset, test_type=test_type) obj_ids = range(1, dp['obj_count'] + 1) scene_ids = range(1, dp['scene_count'] + 1) # Subset of images to be considered if use_image_subset: im_ids_sets = inout.load_yaml(dp['test_set_fpath']) else: im_ids_sets = None # Set threshold of correctness (might be different for each object) error_threshs = {} if error_type in ['add', 'adi']: # Relative to object diameter models_info = inout.load_yaml(dp['models_info_path']) for obj_id in obj_ids: obj_diameter = models_info[obj_id]['diameter'] error_threshs[obj_id] = error_thresh_fact[error_type] *\ obj_diameter else: # The same threshold for all objects for obj_id in obj_ids: error_threshs[obj_id] = error_thresh[error_type] # Go through the test scenes and match estimated poses to GT poses #----------------------------------------------------------------------- matches = [] # Stores info about the matching estimate for each GT for scene_id in scene_ids: # Load GT poses gts = inout.load_gt(dp['scene_gt_mpath'].format(scene_id)) # Load statistics (e.g. visibility fraction) of the GT poses gt_stats_path = dp['scene_gt_stats_mpath'].format( scene_id, visib_delta) gt_stats = inout.load_yaml(gt_stats_path) # Keep the GT poses and their stats only for the selected images if im_ids_sets is not None: im_ids = im_ids_sets[scene_id] gts = {im_id: gts[im_id] for im_id in im_ids} gt_stats = {im_id: gt_stats[im_id] for im_id in im_ids} # Load pre-calculated errors of the pose estimates scene_errs_path = errors_mpath.format(error_path=error_path, scene_id=scene_id) if os.path.isfile(scene_errs_path): errs = inout.load_errors(scene_errs_path) matches += match_poses(gts, gt_stats, errs, scene_id, visib_gt_min, error_threshs, n_top) elif require_all_errors: raise IOError( '{} is missing, but errors for all scenes are required' ' (require_all_results = True).'.format(scene_errs_path)) # Calculate the performance scores #----------------------------------------------------------------------- # Split the dataset of Hinterstoisser to the original LINEMOD dataset # and the Occlusion dataset by TUD (i.e. the extended GT for scene #2) if dataset == 'hinterstoisser': print('-- LINEMOD dataset') eval_sign_lm = 'linemod_' + eval_sign matches_lm = [m for m in matches if m['scene_id'] == m['obj_id']] scores_lm = calc_scores(scene_ids, obj_ids, matches_lm, n_top) # Save scores scores_lm_path = scores_mpath.format(error_path=error_path, eval_sign=eval_sign_lm) inout.save_yaml(scores_lm_path, scores_lm) # Save matches matches_path = matches_mpath.format(error_path=error_path, eval_sign=eval_sign_lm) inout.save_yaml(matches_path, matches_lm) print('-- Occlusion dataset') eval_sign_occ = 'occlusion_' + eval_sign matches_occ = [m for m in matches if m['scene_id'] == 2] scene_ids_occ = [2] obj_ids_occ = [1, 2, 5, 6, 8, 9, 10, 11, 12] scores_occ = calc_scores(scene_ids_occ, obj_ids_occ, matches_occ, n_top) # Save scores scores_occ_path = scores_mpath.format(error_path=error_path, eval_sign=eval_sign_occ) inout.save_yaml(scores_occ_path, scores_occ) # Save matches matches_path = matches_mpath.format(error_path=error_path, eval_sign=eval_sign_occ) inout.save_yaml(matches_path, matches_occ) else: scores = calc_scores(scene_ids, obj_ids, matches, n_top) # Save scores scores_path = scores_mpath.format(error_path=error_path, eval_sign=eval_sign) inout.save_yaml(scores_path, scores) # Save matches matches_path = matches_mpath.format(error_path=error_path, eval_sign=eval_sign) inout.save_yaml(matches_path, matches) print('Done.')
vis = 0.5 * depth_im_vis + 0.5 * visib_gt_vis vis[vis > 1] = 1 vis_path = vis_mpath.format( dataset, delta, data_id, im_id, gt_id) inout.save_im(vis_path, vis) # Mask of depth differences below delta # mask_below_delta_vis = np.dstack([mask_below_delta, # zero_ch, zero_ch]) # vis_delta = 0.5 * depth_im_vis + 0.5 * mask_below_delta_vis # vis_delta[vis_delta > 1] = 1 # vis_delta_path = vis_delta_mpath.format( # dataset, delta, data_id, im_id, gt_id, delta) # inout.save_im(vis_delta_path, vis_delta) res_path = dp[gt_stats_mpath_key].format(data_id, delta) misc.ensure_dir(os.path.dirname(res_path)) inout.save_yaml(res_path, gt_stats) # visib_to_below_delta_fracs = sorted(visib_to_below_delta_fracs, # key=lambda x: x['frac'], reverse=True) # for i in range(200): # e = visib_to_below_delta_fracs[i] # print('{}: data_id: {}, im_id: {}, gt_id: {}, frac: {}'.format( # i, e['data_id'], e['im_id'], e['gt_id'], e['frac'] # )) # # import matplotlib.pyplot as plt # plt.plot([e['frac'] for e in visib_to_below_delta_fracs]) # plt.show()
rgb = cv2.resize(rgb, p['cam']['im_size'], interpolation=cv2.INTER_AREA) # rgb = scipy.misc.imresize(rgb, par['cam']['im_size'][::-1], 'bicubic') # Save the rendered images inout.save_im(out_rgb_mpath.format(obj_id, im_id), rgb) inout.save_depth(out_depth_mpath.format(obj_id, im_id), depth) # Get 2D bounding box of the object model at the ground truth pose ys, xs = np.nonzero(depth > 0) obj_bb = misc.calc_2d_bbox(xs, ys, p['cam']['im_size']) obj_info[im_id] = { 'cam_K': p['cam']['K'].flatten().tolist(), 'view_level': int(views_level[view_id]), # 'sphere_radius': float(radius) } obj_gt[im_id] = [{ 'cam_R_m2c': view['R'].flatten().tolist(), 'cam_t_m2c': view['t'].flatten().tolist(), 'obj_bb': [int(x) for x in obj_bb], 'obj_id': int(obj_id) }] im_id += 1 # Save metadata inout.save_yaml(out_obj_info_path.format(obj_id), obj_info) inout.save_yaml(out_obj_gt_path.format(obj_id), obj_gt) print("end of line for break points")
def main(): # Paths to pose errors (calculated using eval_calc_errors.py) # --------------------------------------------------------------------------- error_bpath = "/path/to/eval/" error_paths = [ pjoin(error_bpath, "hodan-iros15_hinterstoisser"), # pjoin(error_bpath, 'hodan-iros15_tless_primesense'), ] error_dir = "error:vsd_ntop:1_delta:15_tau:20_cost:step" for i in range(len(error_paths)): error_paths[i] = os.path.join(error_paths[i], error_dir) # Other paths # --------------------------------------------------------------------------- # Mask of path to the input file with calculated errors errors_mpath = pjoin("{error_path}", "errors_{scene_id:02d}.yml") # Mask of path to the output file with established matches and calculated scores matches_mpath = pjoin("{error_path}", "matches_{eval_sign}.yml") scores_mpath = pjoin("{error_path}", "scores_{eval_sign}.yml") # Parameters # --------------------------------------------------------------------------- use_image_subset = True # Whether to use the specified subset of images require_all_errors = True # Whether to break if some errors are missing visib_gt_min = 0.1 # Minimum visible surface fraction of valid GT pose visib_delta = 15 # [mm] # Threshold of correctness error_thresh = { "vsd": 0.3, "cou": 0.5, "te": 5.0, "re": 5.0 } # [cm] # [deg] # Factor k; threshold of correctness = k * d, where d is the object diameter error_thresh_fact = {"add": 0.1, "adi": 0.1} # Evaluation # --------------------------------------------------------------------------- for error_path in error_paths: # Parse info about the errors from the folder names error_sign = os.path.basename(error_path) error_type = error_sign.split("_")[0].split(":")[1] n_top = int(error_sign.split("_")[1].split(":")[1]) res_sign = os.path.basename(os.path.dirname(error_path)).split("_") method = res_sign[0] dataset = res_sign[1] test_type = res_sign[2] if len(res_sign) > 3 else "" # Evaluation signature if error_type in ["add", "adi"]: eval_sign = "thf:" + str(error_thresh_fact[error_type]) else: eval_sign = "th:" + str(error_thresh[error_type]) eval_sign += "_min-visib:" + str(visib_gt_min) print("--- Processing: {}, {}, {}".format(method, dataset, error_type)) # Load dataset parameters dp = get_dataset_params(dataset, test_type=test_type) obj_ids = range(1, dp["obj_count"] + 1) scene_ids = range(1, dp["scene_count"] + 1) # Subset of images to be considered if use_image_subset: im_ids_sets = inout.load_yaml(dp["test_set_fpath"]) else: im_ids_sets = None # Set threshold of correctness (might be different for each object) error_threshs = {} if error_type in ["add", "adi"]: # Relative to object diameter models_info = inout.load_yaml(dp["models_info_path"]) for obj_id in obj_ids: obj_diameter = models_info[obj_id]["diameter"] error_threshs[ obj_id] = error_thresh_fact[error_type] * obj_diameter else: # The same threshold for all objects for obj_id in obj_ids: error_threshs[obj_id] = error_thresh[error_type] # Go through the test scenes and match estimated poses to GT poses # ----------------------------------------------------------------------- matches = [] # Stores info about the matching estimate for each GT for scene_id in scene_ids: # Load GT poses gts = inout.load_gt(dp["scene_gt_mpath"].format(scene_id)) # Load statistics (e.g. visibility fraction) of the GT poses gt_stats_path = dp["scene_gt_stats_mpath"].format( scene_id, visib_delta) gt_stats = inout.load_yaml(gt_stats_path) # Keep the GT poses and their stats only for the selected images if im_ids_sets is not None: im_ids = im_ids_sets[scene_id] gts = {im_id: gts[im_id] for im_id in im_ids} gt_stats = {im_id: gt_stats[im_id] for im_id in im_ids} # Load pre-calculated errors of the pose estimates scene_errs_path = errors_mpath.format(error_path=error_path, scene_id=scene_id) if os.path.isfile(scene_errs_path): errs = inout.load_errors(scene_errs_path) matches += match_poses(gts, gt_stats, errs, scene_id, visib_gt_min, error_threshs, n_top) elif require_all_errors: raise IOError( "{} is missing, but errors for all scenes are required" " (require_all_results = True).".format(scene_errs_path)) # Calculate the performance scores # ----------------------------------------------------------------------- # Split the dataset of Hinterstoisser to the original LINEMOD dataset # and the Occlusion dataset by TUD (i.e. the extended GT for scene #2) if dataset == "hinterstoisser": print("-- LINEMOD dataset") eval_sign_lm = "linemod_" + eval_sign matches_lm = [m for m in matches if m["scene_id"] == m["obj_id"]] scores_lm = calc_scores(scene_ids, obj_ids, matches_lm, n_top) # Save scores scores_lm_path = scores_mpath.format(error_path=error_path, eval_sign=eval_sign_lm) inout.save_yaml(scores_lm_path, scores_lm) # Save matches matches_path = matches_mpath.format(error_path=error_path, eval_sign=eval_sign_lm) inout.save_yaml(matches_path, matches_lm) print("-- Occlusion dataset") eval_sign_occ = "occlusion_" + eval_sign matches_occ = [m for m in matches if m["scene_id"] == 2] scene_ids_occ = [2] obj_ids_occ = [1, 2, 5, 6, 8, 9, 10, 11, 12] scores_occ = calc_scores(scene_ids_occ, obj_ids_occ, matches_occ, n_top) # Save scores scores_occ_path = scores_mpath.format(error_path=error_path, eval_sign=eval_sign_occ) inout.save_yaml(scores_occ_path, scores_occ) # Save matches matches_path = matches_mpath.format(error_path=error_path, eval_sign=eval_sign_occ) inout.save_yaml(matches_path, matches_occ) else: scores = calc_scores(scene_ids, obj_ids, matches, n_top) # Save scores scores_path = scores_mpath.format(error_path=error_path, eval_sign=eval_sign) inout.save_yaml(scores_path, scores) # Save matches matches_path = matches_mpath.format(error_path=error_path, eval_sign=eval_sign) inout.save_yaml(matches_path, matches) print("Done.")