def main(): # Paths to pose errors (calculated using eval_calc_errors.py) #--------------------------------------------------------------------------- top_level_path = os.path.dirname(os.path.dirname( os.path.abspath(__file__))) dataset = 'hinterstoisser' # dataset = 'tless' # dataset = 'tudlight' # dataset = 'rutgers' # dataset = 'tejani' # dataset = 'doumanoglou' # dataset = 'toyotalight' error_bpath = pjoin(top_level_path, 'eval') error_paths = [ pjoin(error_bpath, 'patch-linemod_' + dataset), # pjoin(error_bpath, 'hodan-iros15_tless_primesense'), ] error_dir = 'error=vsd_ntop=1_delta=15_tau=20_cost=step' for i in range(len(error_paths)): error_paths[i] = os.path.join(error_paths[i], error_dir) # Other paths #--------------------------------------------------------------------------- # Mask of path to the input file with calculated errors errors_mpath = pjoin('{error_path}', 'errors_{scene_id:02d}.yml') # Mask of path to the output file with established matches and calculated scores matches_mpath = pjoin('{error_path}', 'matches_{eval_sign}.yml') scores_mpath = pjoin('{error_path}', 'scores_{eval_sign}.yml') # Parameters #--------------------------------------------------------------------------- use_image_subset = True # Whether to use the specified subset of images require_all_errors = True # Whether to break if some errors are missing visib_gt_min = 0.1 # Minimum visible surface fraction of valid GT pose visib_delta = 15 # [mm] # Threshold of correctness error_thresh = { 'vsd': 0.3, 'cou': 0.5, 'te': 5.0, # [cm] 're': 5.0 # [deg] } # Factor k; threshold of correctness = k * d, where d is the object diameter error_thresh_fact = {'add': 0.1, 'adi': 0.1} # Evaluation #--------------------------------------------------------------------------- for error_path in error_paths: # Parse info about the errors from the folder names error_sign = os.path.basename(error_path) error_type = error_sign.split('_')[0].split('=')[1] n_top = int(error_sign.split('_')[1].split('=')[1]) res_sign = os.path.basename(os.path.dirname(error_path)).split('_') method = res_sign[0] dataset = res_sign[1] test_type = res_sign[2] if len(res_sign) > 3 else '' # Evaluation signature if error_type in ['add', 'adi']: eval_sign = 'thf=' + str(error_thresh_fact[error_type]) else: eval_sign = 'th=' + str(error_thresh[error_type]) eval_sign += '_min-visib=' + str(visib_gt_min) print('--- Processing: {}, {}, {}'.format(method, dataset, error_type)) # Load dataset parameters dp = get_dataset_params(dataset, test_type=test_type) obj_ids = range(1, dp['obj_count'] + 1) scene_ids = range(1, dp['scene_count'] + 1) # Subset of images to be considered if use_image_subset: im_ids_sets = inout.load_yaml(dp['test_set_fpath']) else: im_ids_sets = None # Set threshold of correctness (might be different for each object) error_threshs = {} if error_type in ['add', 'adi']: # Relative to object diameter models_info = inout.load_yaml(dp['models_info_path']) for obj_id in obj_ids: obj_diameter = models_info[obj_id]['diameter'] error_threshs[obj_id] = error_thresh_fact[error_type] *\ obj_diameter else: # The same threshold for all objects for obj_id in obj_ids: error_threshs[obj_id] = error_thresh[error_type] # Go through the test scenes and match estimated poses to GT poses #----------------------------------------------------------------------- matches = [] # Stores info about the matching estimate for each GT for scene_id in scene_ids: # Load GT poses gts = inout.load_gt(dp['scene_gt_mpath'].format(scene_id)) # Load statistics (e.g. visibility fraction) of the GT poses gt_stats_path = dp['scene_gt_stats_mpath'].format( scene_id, visib_delta) gt_stats = inout.load_yaml(gt_stats_path) # Keep the GT poses and their stats only for the selected images if im_ids_sets is not None: im_ids = im_ids_sets[scene_id] gts = {im_id: gts[im_id] for im_id in im_ids} gt_stats = {im_id: gt_stats[im_id] for im_id in im_ids} # Load pre-calculated errors of the pose estimates scene_errs_path = errors_mpath.format(error_path=error_path, scene_id=scene_id) if os.path.isfile(scene_errs_path): errs = inout.load_errors(scene_errs_path) matches += match_poses(gts, gt_stats, errs, scene_id, visib_gt_min, error_threshs, n_top) elif require_all_errors: raise IOError( '{} is missing, but errors for all scenes are required' ' (require_all_results = True).'.format(scene_errs_path)) # Calculate the performance scores #----------------------------------------------------------------------- # Split the dataset of Hinterstoisser to the original LINEMOD dataset # and the Occlusion dataset by TUD (i.e. the extended GT for scene #2) if dataset == 'hinterstoisser': print('-- LINEMOD dataset') eval_sign_lm = 'linemod_' + eval_sign matches_lm = [m for m in matches if m['scene_id'] == m['obj_id']] scores_lm = calc_scores(scene_ids, obj_ids, matches_lm, n_top) # Save scores scores_lm_path = scores_mpath.format(error_path=error_path, eval_sign=eval_sign_lm) inout.save_yaml(scores_lm_path, scores_lm) # Save matches matches_path = matches_mpath.format(error_path=error_path, eval_sign=eval_sign_lm) inout.save_yaml(matches_path, matches_lm) print('-- Occlusion dataset') eval_sign_occ = 'occlusion_' + eval_sign matches_occ = [m for m in matches if m['scene_id'] == 2] scene_ids_occ = [2] obj_ids_occ = [1, 2, 5, 6, 8, 9, 10, 11, 12] scores_occ = calc_scores(scene_ids_occ, obj_ids_occ, matches_occ, n_top) # Save scores scores_occ_path = scores_mpath.format(error_path=error_path, eval_sign=eval_sign_occ) inout.save_yaml(scores_occ_path, scores_occ) # Save matches matches_path = matches_mpath.format(error_path=error_path, eval_sign=eval_sign_occ) inout.save_yaml(matches_path, matches_occ) else: scores = calc_scores(scene_ids, obj_ids, matches, n_top) # Save scores scores_path = scores_mpath.format(error_path=error_path, eval_sign=eval_sign) inout.save_yaml(scores_path, scores) # Save matches matches_path = matches_mpath.format(error_path=error_path, eval_sign=eval_sign) inout.save_yaml(matches_path, matches) print('Done.')
print('Matching: {}, {}, {}, {}, {}'.format(error_type, method, dataset, test_type, scene_id)) # Load GT poses gts = inout.load_gt(dp['scene_gt_mpath'].format(scene_id)) # Load visibility fractions of the GT poses gt_visib_path = dp['scene_gt_stats_mpath'].format( scene_id, visib_delta) gt_visib = inout.load_yaml(gt_visib_path) # Load pre-calculated errors of the pose estimates scene_errs_path = errors_mpath.format(error_path=error_path, scene_id=scene_id) errs = inout.load_errors(scene_errs_path) # Organize the errors by image id and object id (for faster query) errs_org = {} for e in errs: errs_org.setdefault(e['im_id'], {}).\ setdefault(e['obj_id'], []).append(e) # Matching for im_id, gts_im in gts.items(): matches_im = [{ 'scene_id': scene_id, 'im_id': im_id, 'obj_id': gt['obj_id'], 'gt_id': gt_id, 'est_id': -1,
def main(): # Paths to pose errors (calculated using eval_calc_errors.py) # --------------------------------------------------------------------------- error_bpath = "/path/to/eval/" error_paths = [ pjoin(error_bpath, "hodan-iros15_hinterstoisser"), # pjoin(error_bpath, 'hodan-iros15_tless_primesense'), ] error_dir = "error:vsd_ntop:1_delta:15_tau:20_cost:step" for i in range(len(error_paths)): error_paths[i] = os.path.join(error_paths[i], error_dir) # Other paths # --------------------------------------------------------------------------- # Mask of path to the input file with calculated errors errors_mpath = pjoin("{error_path}", "errors_{scene_id:02d}.yml") # Mask of path to the output file with established matches and calculated scores matches_mpath = pjoin("{error_path}", "matches_{eval_sign}.yml") scores_mpath = pjoin("{error_path}", "scores_{eval_sign}.yml") # Parameters # --------------------------------------------------------------------------- use_image_subset = True # Whether to use the specified subset of images require_all_errors = True # Whether to break if some errors are missing visib_gt_min = 0.1 # Minimum visible surface fraction of valid GT pose visib_delta = 15 # [mm] # Threshold of correctness error_thresh = { "vsd": 0.3, "cou": 0.5, "te": 5.0, "re": 5.0 } # [cm] # [deg] # Factor k; threshold of correctness = k * d, where d is the object diameter error_thresh_fact = {"add": 0.1, "adi": 0.1} # Evaluation # --------------------------------------------------------------------------- for error_path in error_paths: # Parse info about the errors from the folder names error_sign = os.path.basename(error_path) error_type = error_sign.split("_")[0].split(":")[1] n_top = int(error_sign.split("_")[1].split(":")[1]) res_sign = os.path.basename(os.path.dirname(error_path)).split("_") method = res_sign[0] dataset = res_sign[1] test_type = res_sign[2] if len(res_sign) > 3 else "" # Evaluation signature if error_type in ["add", "adi"]: eval_sign = "thf:" + str(error_thresh_fact[error_type]) else: eval_sign = "th:" + str(error_thresh[error_type]) eval_sign += "_min-visib:" + str(visib_gt_min) print("--- Processing: {}, {}, {}".format(method, dataset, error_type)) # Load dataset parameters dp = get_dataset_params(dataset, test_type=test_type) obj_ids = range(1, dp["obj_count"] + 1) scene_ids = range(1, dp["scene_count"] + 1) # Subset of images to be considered if use_image_subset: im_ids_sets = inout.load_yaml(dp["test_set_fpath"]) else: im_ids_sets = None # Set threshold of correctness (might be different for each object) error_threshs = {} if error_type in ["add", "adi"]: # Relative to object diameter models_info = inout.load_yaml(dp["models_info_path"]) for obj_id in obj_ids: obj_diameter = models_info[obj_id]["diameter"] error_threshs[ obj_id] = error_thresh_fact[error_type] * obj_diameter else: # The same threshold for all objects for obj_id in obj_ids: error_threshs[obj_id] = error_thresh[error_type] # Go through the test scenes and match estimated poses to GT poses # ----------------------------------------------------------------------- matches = [] # Stores info about the matching estimate for each GT for scene_id in scene_ids: # Load GT poses gts = inout.load_gt(dp["scene_gt_mpath"].format(scene_id)) # Load statistics (e.g. visibility fraction) of the GT poses gt_stats_path = dp["scene_gt_stats_mpath"].format( scene_id, visib_delta) gt_stats = inout.load_yaml(gt_stats_path) # Keep the GT poses and their stats only for the selected images if im_ids_sets is not None: im_ids = im_ids_sets[scene_id] gts = {im_id: gts[im_id] for im_id in im_ids} gt_stats = {im_id: gt_stats[im_id] for im_id in im_ids} # Load pre-calculated errors of the pose estimates scene_errs_path = errors_mpath.format(error_path=error_path, scene_id=scene_id) if os.path.isfile(scene_errs_path): errs = inout.load_errors(scene_errs_path) matches += match_poses(gts, gt_stats, errs, scene_id, visib_gt_min, error_threshs, n_top) elif require_all_errors: raise IOError( "{} is missing, but errors for all scenes are required" " (require_all_results = True).".format(scene_errs_path)) # Calculate the performance scores # ----------------------------------------------------------------------- # Split the dataset of Hinterstoisser to the original LINEMOD dataset # and the Occlusion dataset by TUD (i.e. the extended GT for scene #2) if dataset == "hinterstoisser": print("-- LINEMOD dataset") eval_sign_lm = "linemod_" + eval_sign matches_lm = [m for m in matches if m["scene_id"] == m["obj_id"]] scores_lm = calc_scores(scene_ids, obj_ids, matches_lm, n_top) # Save scores scores_lm_path = scores_mpath.format(error_path=error_path, eval_sign=eval_sign_lm) inout.save_yaml(scores_lm_path, scores_lm) # Save matches matches_path = matches_mpath.format(error_path=error_path, eval_sign=eval_sign_lm) inout.save_yaml(matches_path, matches_lm) print("-- Occlusion dataset") eval_sign_occ = "occlusion_" + eval_sign matches_occ = [m for m in matches if m["scene_id"] == 2] scene_ids_occ = [2] obj_ids_occ = [1, 2, 5, 6, 8, 9, 10, 11, 12] scores_occ = calc_scores(scene_ids_occ, obj_ids_occ, matches_occ, n_top) # Save scores scores_occ_path = scores_mpath.format(error_path=error_path, eval_sign=eval_sign_occ) inout.save_yaml(scores_occ_path, scores_occ) # Save matches matches_path = matches_mpath.format(error_path=error_path, eval_sign=eval_sign_occ) inout.save_yaml(matches_path, matches_occ) else: scores = calc_scores(scene_ids, obj_ids, matches, n_top) # Save scores scores_path = scores_mpath.format(error_path=error_path, eval_sign=eval_sign) inout.save_yaml(scores_path, scores) # Save matches matches_path = matches_mpath.format(error_path=error_path, eval_sign=eval_sign) inout.save_yaml(matches_path, matches) print("Done.")