else: # 'test' data_ids = range(1, dp['scene_count'] + 1) gt_mpath_key = 'scene_gt_mpath' gt_stats_mpath_key = 'scene_gt_stats_mpath' # Subset of images to be considered if dataset_part == 'test' and use_image_subset: im_ids_sets = inout.load_yaml(dp['test_set_fpath']) else: im_ids_sets = None # Load the GT statistics gt_stats = [] for data_id in data_ids: print('Loading GT stats: {}, {}'.format(dataset, data_id)) gts = inout.load_gt(dp[gt_mpath_key].format(data_id)) gt_stats_curr = inout.load_yaml(dp[gt_stats_mpath_key].format( data_id, delta)) # Considered subset of images for the current scene if im_ids_sets is not None: im_ids_curr = im_ids_sets[data_id] else: im_ids_curr = sorted(gt_stats_curr.keys()) for im_id in im_ids_curr: gt_stats_im = gt_stats_curr[im_id] for gt_id, p in enumerate(gt_stats_im): p['data_id'] = data_id p['im_id'] = im_id p['gt_id'] = gt_id
obj_id_in_scene_array = list() obj_id_in_scene_array.append(scene_id) if dataset =='doumanoglou' and scene_id == 3: obj_id_in_scene_array = [1, 2] if dataset == 'hinterstoisser' and scene_id == 2: obj_id_in_scene_array = [1, 2, 5, 6, 8, 9, 10, 11, 12] # for occ dataset for obj_id_in_scene in obj_id_in_scene_array: # Load scene info and gt poses print('#' * 20) print('\nreading detector template & info, obj: {}'.format(obj_id_in_scene)) misc.ensure_dir(join(result_base_path, '{:02d}'.format(scene_id))) scene_info = inout.load_info(dp['scene_info_mpath'].format(scene_id)) scene_gt = inout.load_gt(dp['scene_gt_mpath'].format(scene_id)) model = inout.load_ply(dp['model_mpath'].format(obj_id_in_scene)) ###################################################### # prepare renderer rather than rebuilding every time clip_near = 10 # [mm] clip_far = 10000 # [mm] ambient_weight = 0.8 surf_color = None mode = 'rgb+depth' K = dp['cam']['K'] shading = 'phong' # Load model texture
def main(): # Paths to pose errors (calculated using eval_calc_errors.py) #--------------------------------------------------------------------------- top_level_path = os.path.dirname(os.path.dirname( os.path.abspath(__file__))) dataset = 'hinterstoisser' # dataset = 'tless' # dataset = 'tudlight' # dataset = 'rutgers' # dataset = 'tejani' # dataset = 'doumanoglou' # dataset = 'toyotalight' error_bpath = pjoin(top_level_path, 'eval') error_paths = [ pjoin(error_bpath, 'patch-linemod_' + dataset), # pjoin(error_bpath, 'hodan-iros15_tless_primesense'), ] error_dir = 'error=vsd_ntop=1_delta=15_tau=20_cost=step' for i in range(len(error_paths)): error_paths[i] = os.path.join(error_paths[i], error_dir) # Other paths #--------------------------------------------------------------------------- # Mask of path to the input file with calculated errors errors_mpath = pjoin('{error_path}', 'errors_{scene_id:02d}.yml') # Mask of path to the output file with established matches and calculated scores matches_mpath = pjoin('{error_path}', 'matches_{eval_sign}.yml') scores_mpath = pjoin('{error_path}', 'scores_{eval_sign}.yml') # Parameters #--------------------------------------------------------------------------- use_image_subset = True # Whether to use the specified subset of images require_all_errors = True # Whether to break if some errors are missing visib_gt_min = 0.1 # Minimum visible surface fraction of valid GT pose visib_delta = 15 # [mm] # Threshold of correctness error_thresh = { 'vsd': 0.3, 'cou': 0.5, 'te': 5.0, # [cm] 're': 5.0 # [deg] } # Factor k; threshold of correctness = k * d, where d is the object diameter error_thresh_fact = {'add': 0.1, 'adi': 0.1} # Evaluation #--------------------------------------------------------------------------- for error_path in error_paths: # Parse info about the errors from the folder names error_sign = os.path.basename(error_path) error_type = error_sign.split('_')[0].split('=')[1] n_top = int(error_sign.split('_')[1].split('=')[1]) res_sign = os.path.basename(os.path.dirname(error_path)).split('_') method = res_sign[0] dataset = res_sign[1] test_type = res_sign[2] if len(res_sign) > 3 else '' # Evaluation signature if error_type in ['add', 'adi']: eval_sign = 'thf=' + str(error_thresh_fact[error_type]) else: eval_sign = 'th=' + str(error_thresh[error_type]) eval_sign += '_min-visib=' + str(visib_gt_min) print('--- Processing: {}, {}, {}'.format(method, dataset, error_type)) # Load dataset parameters dp = get_dataset_params(dataset, test_type=test_type) obj_ids = range(1, dp['obj_count'] + 1) scene_ids = range(1, dp['scene_count'] + 1) # Subset of images to be considered if use_image_subset: im_ids_sets = inout.load_yaml(dp['test_set_fpath']) else: im_ids_sets = None # Set threshold of correctness (might be different for each object) error_threshs = {} if error_type in ['add', 'adi']: # Relative to object diameter models_info = inout.load_yaml(dp['models_info_path']) for obj_id in obj_ids: obj_diameter = models_info[obj_id]['diameter'] error_threshs[obj_id] = error_thresh_fact[error_type] *\ obj_diameter else: # The same threshold for all objects for obj_id in obj_ids: error_threshs[obj_id] = error_thresh[error_type] # Go through the test scenes and match estimated poses to GT poses #----------------------------------------------------------------------- matches = [] # Stores info about the matching estimate for each GT for scene_id in scene_ids: # Load GT poses gts = inout.load_gt(dp['scene_gt_mpath'].format(scene_id)) # Load statistics (e.g. visibility fraction) of the GT poses gt_stats_path = dp['scene_gt_stats_mpath'].format( scene_id, visib_delta) gt_stats = inout.load_yaml(gt_stats_path) # Keep the GT poses and their stats only for the selected images if im_ids_sets is not None: im_ids = im_ids_sets[scene_id] gts = {im_id: gts[im_id] for im_id in im_ids} gt_stats = {im_id: gt_stats[im_id] for im_id in im_ids} # Load pre-calculated errors of the pose estimates scene_errs_path = errors_mpath.format(error_path=error_path, scene_id=scene_id) if os.path.isfile(scene_errs_path): errs = inout.load_errors(scene_errs_path) matches += match_poses(gts, gt_stats, errs, scene_id, visib_gt_min, error_threshs, n_top) elif require_all_errors: raise IOError( '{} is missing, but errors for all scenes are required' ' (require_all_results = True).'.format(scene_errs_path)) # Calculate the performance scores #----------------------------------------------------------------------- # Split the dataset of Hinterstoisser to the original LINEMOD dataset # and the Occlusion dataset by TUD (i.e. the extended GT for scene #2) if dataset == 'hinterstoisser': print('-- LINEMOD dataset') eval_sign_lm = 'linemod_' + eval_sign matches_lm = [m for m in matches if m['scene_id'] == m['obj_id']] scores_lm = calc_scores(scene_ids, obj_ids, matches_lm, n_top) # Save scores scores_lm_path = scores_mpath.format(error_path=error_path, eval_sign=eval_sign_lm) inout.save_yaml(scores_lm_path, scores_lm) # Save matches matches_path = matches_mpath.format(error_path=error_path, eval_sign=eval_sign_lm) inout.save_yaml(matches_path, matches_lm) print('-- Occlusion dataset') eval_sign_occ = 'occlusion_' + eval_sign matches_occ = [m for m in matches if m['scene_id'] == 2] scene_ids_occ = [2] obj_ids_occ = [1, 2, 5, 6, 8, 9, 10, 11, 12] scores_occ = calc_scores(scene_ids_occ, obj_ids_occ, matches_occ, n_top) # Save scores scores_occ_path = scores_mpath.format(error_path=error_path, eval_sign=eval_sign_occ) inout.save_yaml(scores_occ_path, scores_occ) # Save matches matches_path = matches_mpath.format(error_path=error_path, eval_sign=eval_sign_occ) inout.save_yaml(matches_path, matches_occ) else: scores = calc_scores(scene_ids, obj_ids, matches, n_top) # Save scores scores_path = scores_mpath.format(error_path=error_path, eval_sign=eval_sign) inout.save_yaml(scores_path, scores) # Save matches matches_path = matches_mpath.format(error_path=error_path, eval_sign=eval_sign) inout.save_yaml(matches_path, matches) print('Done.')
if error_type in ['vsd', 'add', 'adi', 'cou']: print('Loading object models...') models = {} for obj_id in range(1, dp['obj_count'] + 1): models[obj_id] = inout.load_ply(dp['model_mpath'].format(obj_id)) # Directories with results for individual scenes scene_dirs = sorted([d for d in glob.glob(os.path.join(result_path, '*')) if os.path.isdir(d)]) for scene_dir in scene_dirs: scene_id = int(os.path.basename(scene_dir)) # Load info and GT poses for the current scene scene_info = inout.load_info(dp['scene_info_mpath'].format(scene_id)) scene_gt = inout.load_gt(dp['scene_gt_mpath'].format(scene_id)) res_paths = sorted(glob.glob(os.path.join(scene_dir, '*.yml'))) errs = [] im_id = -1 depth_im = None for res_id, res_path in enumerate(res_paths): # t = time.time() # Parse image ID and object ID from the filename filename = os.path.basename(res_path).split('.')[0] im_id_prev = im_id im_id, obj_id = map(int, filename.split('_')) if res_id % 10 == 0: dataset_str = dataset
def load_scene(scene_id): """ Loads the specified scene. """ clean() global ref_im_ind global scene_info global scene_gt # Load scene info scene_info_path = par['scene_info_mpath'].format(scene_id) print('Loading scene info: ' + scene_info_path) scene_info = inout.load_info(scene_info_path) ref_im_id = sorted(scene_info.keys())[ref_im_ind] scene_info_ref = scene_info[ref_im_id] R_w2c = scene_info_ref['cam_R_w2c'] R_w2c_inv = np.linalg.inv(R_w2c) t_w2c = scene_info_ref['cam_t_w2c'] # Load ground truth poses for the reference camera coordinate system scene_gt_path = par['scene_gt_mpath'].format(scene_id) print('Loading GT poses: ' + scene_gt_path) scene_gt = inout.load_gt(scene_gt_path) scene_gt_ref = scene_gt[ref_im_id] # Load scene model #bpy.ops.import_mesh.ply(filepath=scene_model_mpath.format(scene_id)) # Load models of objects that are present in the scene for gt in scene_gt_ref: model_path = par['model_mpath'].format(gt['obj_id']) print('Loading model {}: {}'.format(gt['obj_id'], model_path)) bpy.ops.import_mesh.ply(filepath=model_path) # Take poses from the reference camera coordinate system and transform them # to the world coordinate system (using the known camera-to-world trans.) objs = list(bpy.data.objects) objs_moved = [False for _ in objs] for gt in scene_gt_ref: for i, obj in enumerate(objs): obj_id = int(obj.name.split('obj_')[1].split('.')[0]) if gt['obj_id'] == obj_id and not objs_moved[i]: print('Setting pose of model {}'.format(gt['obj_id'])) #print(i, gt, obj_id, objs_moved[i], obj.location) R_m2w = R_w2c_inv.dot(gt['cam_R_m2c']) t_m2w = R_w2c_inv.dot(gt['cam_t_m2c']) - R_w2c_inv.dot(t_w2c) obj.rotation_euler = transform.euler_from_matrix(R_m2w, 'sxyz') obj.location = t_m2w.flatten().tolist() objs_moved[i] = True break # Align view to the loaded elements for area in bpy.context.screen.areas: if area.type == 'VIEW_3D': ctx = bpy.context.copy() ctx['area'] = area ctx['region'] = area.regions[-1] bpy.ops.view3d.view_selected(ctx) # points view # bpy.ops.view3d.camera_to_view_selected(ctx) # points camera print("Scene loaded.")
# 'd}_{:02d}_diff_below_delta={}.jpg' print('Loading object models...') models = {} for obj_id in obj_ids: models[obj_id] = inout.load_ply(dp['model_mpath'].format(obj_id)) # visib_to_below_delta_fracs = [] for data_id in data_ids: if do_vis: misc.ensure_dir(os.path.dirname( vis_mpath.format(dataset, delta, data_id, 0, 0))) # Load scene info and gts info = inout.load_info(dp[info_mpath_key].format(data_id)) gts = inout.load_gt(dp[gt_mpath_key].format(data_id)) # Considered subset of images for the current scene if im_ids_sets is not None: im_ids = im_ids_sets[data_id] else: im_ids = sorted(gts.keys()) gt_stats = {} for im_id in im_ids: print('dataset: {}, scene/obj: {}, im: {}'.format(dataset, data_id, im_id)) K = info[im_id]['cam_K'] depth_path = dp[depth_mpath_key].format(data_id, im_id) depth_im = inout.load_depth(depth_path) depth_im *= dp['cam']['depth_scale'] # to [mm]
def main(): # Paths to pose errors (calculated using eval_calc_errors.py) # --------------------------------------------------------------------------- error_bpath = "/path/to/eval/" error_paths = [ pjoin(error_bpath, "hodan-iros15_hinterstoisser"), # pjoin(error_bpath, 'hodan-iros15_tless_primesense'), ] error_dir = "error:vsd_ntop:1_delta:15_tau:20_cost:step" for i in range(len(error_paths)): error_paths[i] = os.path.join(error_paths[i], error_dir) # Other paths # --------------------------------------------------------------------------- # Mask of path to the input file with calculated errors errors_mpath = pjoin("{error_path}", "errors_{scene_id:02d}.yml") # Mask of path to the output file with established matches and calculated scores matches_mpath = pjoin("{error_path}", "matches_{eval_sign}.yml") scores_mpath = pjoin("{error_path}", "scores_{eval_sign}.yml") # Parameters # --------------------------------------------------------------------------- use_image_subset = True # Whether to use the specified subset of images require_all_errors = True # Whether to break if some errors are missing visib_gt_min = 0.1 # Minimum visible surface fraction of valid GT pose visib_delta = 15 # [mm] # Threshold of correctness error_thresh = { "vsd": 0.3, "cou": 0.5, "te": 5.0, "re": 5.0 } # [cm] # [deg] # Factor k; threshold of correctness = k * d, where d is the object diameter error_thresh_fact = {"add": 0.1, "adi": 0.1} # Evaluation # --------------------------------------------------------------------------- for error_path in error_paths: # Parse info about the errors from the folder names error_sign = os.path.basename(error_path) error_type = error_sign.split("_")[0].split(":")[1] n_top = int(error_sign.split("_")[1].split(":")[1]) res_sign = os.path.basename(os.path.dirname(error_path)).split("_") method = res_sign[0] dataset = res_sign[1] test_type = res_sign[2] if len(res_sign) > 3 else "" # Evaluation signature if error_type in ["add", "adi"]: eval_sign = "thf:" + str(error_thresh_fact[error_type]) else: eval_sign = "th:" + str(error_thresh[error_type]) eval_sign += "_min-visib:" + str(visib_gt_min) print("--- Processing: {}, {}, {}".format(method, dataset, error_type)) # Load dataset parameters dp = get_dataset_params(dataset, test_type=test_type) obj_ids = range(1, dp["obj_count"] + 1) scene_ids = range(1, dp["scene_count"] + 1) # Subset of images to be considered if use_image_subset: im_ids_sets = inout.load_yaml(dp["test_set_fpath"]) else: im_ids_sets = None # Set threshold of correctness (might be different for each object) error_threshs = {} if error_type in ["add", "adi"]: # Relative to object diameter models_info = inout.load_yaml(dp["models_info_path"]) for obj_id in obj_ids: obj_diameter = models_info[obj_id]["diameter"] error_threshs[ obj_id] = error_thresh_fact[error_type] * obj_diameter else: # The same threshold for all objects for obj_id in obj_ids: error_threshs[obj_id] = error_thresh[error_type] # Go through the test scenes and match estimated poses to GT poses # ----------------------------------------------------------------------- matches = [] # Stores info about the matching estimate for each GT for scene_id in scene_ids: # Load GT poses gts = inout.load_gt(dp["scene_gt_mpath"].format(scene_id)) # Load statistics (e.g. visibility fraction) of the GT poses gt_stats_path = dp["scene_gt_stats_mpath"].format( scene_id, visib_delta) gt_stats = inout.load_yaml(gt_stats_path) # Keep the GT poses and their stats only for the selected images if im_ids_sets is not None: im_ids = im_ids_sets[scene_id] gts = {im_id: gts[im_id] for im_id in im_ids} gt_stats = {im_id: gt_stats[im_id] for im_id in im_ids} # Load pre-calculated errors of the pose estimates scene_errs_path = errors_mpath.format(error_path=error_path, scene_id=scene_id) if os.path.isfile(scene_errs_path): errs = inout.load_errors(scene_errs_path) matches += match_poses(gts, gt_stats, errs, scene_id, visib_gt_min, error_threshs, n_top) elif require_all_errors: raise IOError( "{} is missing, but errors for all scenes are required" " (require_all_results = True).".format(scene_errs_path)) # Calculate the performance scores # ----------------------------------------------------------------------- # Split the dataset of Hinterstoisser to the original LINEMOD dataset # and the Occlusion dataset by TUD (i.e. the extended GT for scene #2) if dataset == "hinterstoisser": print("-- LINEMOD dataset") eval_sign_lm = "linemod_" + eval_sign matches_lm = [m for m in matches if m["scene_id"] == m["obj_id"]] scores_lm = calc_scores(scene_ids, obj_ids, matches_lm, n_top) # Save scores scores_lm_path = scores_mpath.format(error_path=error_path, eval_sign=eval_sign_lm) inout.save_yaml(scores_lm_path, scores_lm) # Save matches matches_path = matches_mpath.format(error_path=error_path, eval_sign=eval_sign_lm) inout.save_yaml(matches_path, matches_lm) print("-- Occlusion dataset") eval_sign_occ = "occlusion_" + eval_sign matches_occ = [m for m in matches if m["scene_id"] == 2] scene_ids_occ = [2] obj_ids_occ = [1, 2, 5, 6, 8, 9, 10, 11, 12] scores_occ = calc_scores(scene_ids_occ, obj_ids_occ, matches_occ, n_top) # Save scores scores_occ_path = scores_mpath.format(error_path=error_path, eval_sign=eval_sign_occ) inout.save_yaml(scores_occ_path, scores_occ) # Save matches matches_path = matches_mpath.format(error_path=error_path, eval_sign=eval_sign_occ) inout.save_yaml(matches_path, matches_occ) else: scores = calc_scores(scene_ids, obj_ids, matches, n_top) # Save scores scores_path = scores_mpath.format(error_path=error_path, eval_sign=eval_sign) inout.save_yaml(scores_path, scores) # Save matches matches_path = matches_mpath.format(error_path=error_path, eval_sign=eval_sign) inout.save_yaml(matches_path, matches) print("Done.")