def evaluate(self, res_file, out_dir=None, renderer_type='python'):
"""Evaluates BOP metrics given a result file.
Args:
res_file: Path to the result file.
out_dir: Path to the output directory.
renderer_type: Renderer type. 'python' or 'cpp'.
Returns:
A dictionary holding the results.
Raises:
RuntimeError: If BOP evaluation failed.
"""
if out_dir is None:
out_dir = self._out_dir
ests = inout.load_bop_results(res_file)
ests = [self._convert_pose_to_bop(est) for est in ests]
res_name = os.path.splitext(os.path.basename(res_file))[0]
bop_res_name = 'bop-{}_{}-{}'.format(res_name.replace('_', '-'),
self._setup, self._split)
bop_res_file = os.path.join(out_dir, "{}.csv".format(bop_res_name))
inout.save_bop_results(bop_res_file, ests)
eval_cmd = [
'python',
os.path.join('scripts', 'eval_bop19.py'),
'--renderer_type={}'.format(renderer_type),
'--result_filenames={}'.format(bop_res_file),
'--results_path={}'.format(out_dir),
'--eval_path={}'.format(out_dir),
]
cwd = "bop_toolkit"
env = os.environ.copy()
env['PYTHONPATH'] = "."
env['BOP_PATH'] = self._bop_dir
if subprocess.run(eval_cmd, cwd=cwd, env=env).returncode != 0:
raise RuntimeError('BOP evaluation failed.')
log_file = os.path.join(
out_dir, "bop_eval_{}_{}.log".format(self._name, res_name))
logger = get_logger(log_file)
results = {}
results['all'] = self._derive_bop_results(out_dir, bop_res_name, False,
logger)
results['grasp_only'] = self._derive_bop_results(
out_dir, bop_res_name, True, logger)
logger.info('Evaluation complete.')
return results
# Load the estimation targets.
targets = inout.load_json(
os.path.join(dp_split['base_path'], p['targets_filename']))
# Organize the targets by scene, image and object.
misc.log('Organizing estimation targets...')
targets_org = {}
for target in targets:
targets_org.setdefault(target['scene_id'],
{}).setdefault(target['im_id'],
{})[target['obj_id']] = target
# Load pose estimates.
misc.log('Loading pose estimates...')
ests = inout.load_bop_results(
os.path.join(p['results_path'], result_filename))
# Organize the pose estimates by scene, image and object.
misc.log('Organizing pose estimates...')
ests_org = {}
for est in ests:
ests_org.setdefault(est['scene_id'],
{}).setdefault(est['im_id'],
{}).setdefault(est['obj_id'],
[]).append(est)
for scene_id, scene_targets in targets_org.items():
# Load camera and GT poses for the current scene.
scene_camera = inout.load_scene_camera(
dp_split['scene_camera_tpath'].format(scene_id=scene_id))
misc.log('===========')
misc.log('EVALUATING: {}'.format(result_filename))
misc.log('===========')
time_start = time.time()
# Volume under recall surface (VSD) / area under recall curve (MSSD, MSPD).
average_recalls = {}
# Name of the result and the dataset.
result_name = os.path.splitext(os.path.basename(result_filename))[0]
dataset = str(result_name.split('_')[1].split('-')[0])
# Calculate the average estimation time per image.
ests = inout.load_bop_results(
os.path.join(p['results_path'], result_filename), version='bop19')
times = {}
times_available = True
for est in ests:
result_key = '{:06d}_{:06d}'.format(est['scene_id'], est['im_id'])
if est['time'] < 0:
# All estimation times must be provided.
times_available = False
break
elif result_key in times:
if abs(times[result_key] - est['time']) > 0.001:
raise ValueError(
'The running time for scene {} and image {} is not the same for '
'all estimates.'.format(est['scene_id'], est['im_id']))
else:
times[result_key] = est['time']
p['renderer_type'],
mode=renderer_mode)
# Load object models.
models = {}
for obj_id in dp_model['obj_ids']:
misc.log('Loading 3D model of object {}...'.format(obj_id))
model_path = dp_model['model_tpath'].format(obj_id=obj_id)
model_color = None
if not p['vis_orig_color']:
model_color = tuple(colors[(obj_id - 1) % len(colors)])
ren.add_object(obj_id, model_path, surf_color=model_color)
# Load pose estimates.
misc.log('Loading pose estimates...')
ests = inout.load_bop_results(
os.path.join(config.results_path, result_fname))
# Organize the pose estimates by scene, image and object.
misc.log('Organizing pose estimates...')
ests_org = {}
for est in ests:
ests_org.setdefault(est['scene_id'],
{}).setdefault(est['im_id'],
{}).setdefault(est['obj_id'],
[]).append(est)
for scene_id, scene_ests in ests_org.items():
# Load info and ground-truth poses for the current scene.
scene_camera = inout.load_scene_camera(
dp_split['scene_camera_tpath'].format(scene_id=scene_id))
