def evaluate(self, res_file, out_dir=None, renderer_type='python'): """Evaluates BOP metrics given a result file. Args: res_file: Path to the result file. out_dir: Path to the output directory. renderer_type: Renderer type. 'python' or 'cpp'. Returns: A dictionary holding the results. Raises: RuntimeError: If BOP evaluation failed. """ if out_dir is None: out_dir = self._out_dir ests = inout.load_bop_results(res_file) ests = [self._convert_pose_to_bop(est) for est in ests] res_name = os.path.splitext(os.path.basename(res_file))[0] bop_res_name = 'bop-{}_{}-{}'.format(res_name.replace('_', '-'), self._setup, self._split) bop_res_file = os.path.join(out_dir, "{}.csv".format(bop_res_name)) inout.save_bop_results(bop_res_file, ests) eval_cmd = [ 'python', os.path.join('scripts', 'eval_bop19.py'), '--renderer_type={}'.format(renderer_type), '--result_filenames={}'.format(bop_res_file), '--results_path={}'.format(out_dir), '--eval_path={}'.format(out_dir), ] cwd = "bop_toolkit" env = os.environ.copy() env['PYTHONPATH'] = "." env['BOP_PATH'] = self._bop_dir if subprocess.run(eval_cmd, cwd=cwd, env=env).returncode != 0: raise RuntimeError('BOP evaluation failed.') log_file = os.path.join( out_dir, "bop_eval_{}_{}.log".format(self._name, res_name)) logger = get_logger(log_file) results = {} results['all'] = self._derive_bop_results(out_dir, bop_res_name, False, logger) results['grasp_only'] = self._derive_bop_results( out_dir, bop_res_name, True, logger) logger.info('Evaluation complete.') return results
# Load the estimation targets. targets = inout.load_json( os.path.join(dp_split['base_path'], p['targets_filename'])) # Organize the targets by scene, image and object. misc.log('Organizing estimation targets...') targets_org = {} for target in targets: targets_org.setdefault(target['scene_id'], {}).setdefault(target['im_id'], {})[target['obj_id']] = target # Load pose estimates. misc.log('Loading pose estimates...') ests = inout.load_bop_results( os.path.join(p['results_path'], result_filename)) # Organize the pose estimates by scene, image and object. misc.log('Organizing pose estimates...') ests_org = {} for est in ests: ests_org.setdefault(est['scene_id'], {}).setdefault(est['im_id'], {}).setdefault(est['obj_id'], []).append(est) for scene_id, scene_targets in targets_org.items(): # Load camera and GT poses for the current scene. scene_camera = inout.load_scene_camera( dp_split['scene_camera_tpath'].format(scene_id=scene_id))
misc.log('===========') misc.log('EVALUATING: {}'.format(result_filename)) misc.log('===========') time_start = time.time() # Volume under recall surface (VSD) / area under recall curve (MSSD, MSPD). average_recalls = {} # Name of the result and the dataset. result_name = os.path.splitext(os.path.basename(result_filename))[0] dataset = str(result_name.split('_')[1].split('-')[0]) # Calculate the average estimation time per image. ests = inout.load_bop_results( os.path.join(p['results_path'], result_filename), version='bop19') times = {} times_available = True for est in ests: result_key = '{:06d}_{:06d}'.format(est['scene_id'], est['im_id']) if est['time'] < 0: # All estimation times must be provided. times_available = False break elif result_key in times: if abs(times[result_key] - est['time']) > 0.001: raise ValueError( 'The running time for scene {} and image {} is not the same for ' 'all estimates.'.format(est['scene_id'], est['im_id'])) else: times[result_key] = est['time']
p['renderer_type'], mode=renderer_mode) # Load object models. models = {} for obj_id in dp_model['obj_ids']: misc.log('Loading 3D model of object {}...'.format(obj_id)) model_path = dp_model['model_tpath'].format(obj_id=obj_id) model_color = None if not p['vis_orig_color']: model_color = tuple(colors[(obj_id - 1) % len(colors)]) ren.add_object(obj_id, model_path, surf_color=model_color) # Load pose estimates. misc.log('Loading pose estimates...') ests = inout.load_bop_results( os.path.join(config.results_path, result_fname)) # Organize the pose estimates by scene, image and object. misc.log('Organizing pose estimates...') ests_org = {} for est in ests: ests_org.setdefault(est['scene_id'], {}).setdefault(est['im_id'], {}).setdefault(est['obj_id'], []).append(est) for scene_id, scene_ests in ests_org.items(): # Load info and ground-truth poses for the current scene. scene_camera = inout.load_scene_camera( dp_split['scene_camera_tpath'].format(scene_id=scene_id))