time_start = time.time()
# Parse info about the errors from the folder name.
error_sign = os.path.basename(error_dir_path)
err_type = str(error_sign.split('_')[0].split('=')[1])
n_top = int(error_sign.split('_')[1].split('=')[1])
result_info = os.path.basename(os.path.dirname(error_dir_path)).split('_')
method = result_info[0]
dataset_info = result_info[1].split('-')
dataset = dataset_info[0]
split = dataset_info[1]
split_type = dataset_info[2] if len(dataset_info) > 2 else None
# Evaluation signature.
score_sign = misc.get_score_signature(
p['correct_th'][err_type], p['visib_gt_min'])
misc.log('Calculating score - error: {}, method: {}, dataset: {}.'.format(
err_type, method, dataset))
# Load dataset parameters.
dp_split = dataset_params.get_split_params(
p['datasets_path'], dataset, split, split_type)
model_type = 'eval'
dp_model = dataset_params.get_model_params(
p['datasets_path'], dataset, model_type)
# Load info about the object models.
models_info = inout.load_json(dp_model['models_info_path'], keys_to_int=True)
os.path.join('scripts', 'eval_calc_scores.py'),
'--error_dir_paths={}'.format(error_dir_path),
'--eval_path={}'.format(p['eval_path']),
'--targets_filename={}'.format(p['targets_filename']),
'--visib_gt_min={}'.format(p['visib_gt_min'])
]
calc_scores_cmd += ['--correct_th_{}={}'.format(
error['type'], ','.join(map(str, correct_th)))]
misc.log('Running: ' + ' '.join(calc_scores_cmd))
if subprocess.call(calc_scores_cmd) != 0:
raise RuntimeError('Calculation of scores failed.')
# Path to file with calculated scores.
score_sign = misc.get_score_signature(correct_th, p['visib_gt_min'])
scores_filename = 'scores_{}.json'.format(score_sign)
scores_path = os.path.join(
p['eval_path'], result_name, error_sign, scores_filename)
# Load the scores.
misc.log('Loading calculated scores from: {}'.format(scores_path))
scores = inout.load_json(scores_path)
recalls.append(scores['recall'])
average_recalls[error['type']] = np.mean(recalls)
misc.log('Recall scores: {}'.format(' '.join(map(str, recalls))))
misc.log('Average recall: {}'.format(average_recalls[error['type']]))
def _derive_bop_results(self, out_dir, result_name, grasp_only, logger):
"""Derives BOP results.
Args:
out_dir: Path to the output directory.
result_name: BOP result name. Should be the name of a folder under out_dir
that contains output from BOP evaluation.
grasp_only: Whether to derive results on grasped objects only.
logger: Logger.
Returns:
A dictionary holding the results.
"""
if grasp_only:
set_str = 'grasp only'
else:
set_str = 'all'
logger.info('Deriving results for *{}*'.format(set_str))
average_recalls = {}
average_recalls_obj = defaultdict(lambda: {})
for error in self._p['errors']:
error_dir_paths = {}
if error['type'] == 'vsd':
for vsd_tau in error['vsd_taus']:
error_sign = misc.get_error_signature(
error['type'],
error['n_top'],
vsd_delta=error['vsd_delta'],
vsd_tau=vsd_tau)
error_dir_paths[error_sign] = os.path.join(
result_name, error_sign)
else:
error_sign = misc.get_error_signature(error['type'],
error['n_top'])
error_dir_paths[error_sign] = os.path.join(
result_name, error_sign)
recalls = []
recalls_obj = defaultdict(lambda: [])
for error_sign, error_dir_path in error_dir_paths.items():
for correct_th in error['correct_th']:
score_sign = misc.get_score_signature(
correct_th, self._p['visib_gt_min'])
matches_filename = "matches_{}.json".format(score_sign)
matches_path = os.path.join(out_dir, error_dir_path,
matches_filename)
matches = inout.load_json(matches_path)
if grasp_only:
matches = [
m for m in matches if m['obj_id'] ==
self._grasp_id[m['scene_id']][m['im_id']]
]
scores = score.calc_localization_scores(self._scene_ids,
self._obj_ids,
matches,
error['n_top'],
do_print=False)
recalls.append(scores['recall'])
for i, r in scores['obj_recalls'].items():
recalls_obj[i].append(r)
average_recalls[error['type']] = np.mean(recalls)
for i, r in recalls_obj.items():
average_recalls_obj[i][error['type']] = np.mean(r)
results = {i: r * 100 for i, r in average_recalls.items()}
results['mean'] = np.mean(
[results['vsd'], results['mssd'], results['mspd']])
keys, values = tuple(zip(*results.items()))
table = tabulate(
[values],
headers=keys,
tablefmt='pipe',
floatfmt='.3f',
stralign='center',
numalign='center',
)
logger.info('Evaluation results for *{}*: \n'.format(set_str) + table)
results_per_object = {}
for i, v in average_recalls_obj.items():
res = {k: r * 100 for k, r in v.items()}
res['mean'] = np.mean([res['vsd'], res['mssd'], res['mspd']])
results_per_object[self._dataset.ycb_classes[i]] = res
n_cols = 5
results_tuple = [(k, v['vsd'], v['mssd'], v['mspd'], v['mean'])
for k, v in results_per_object.items()]
results_flatten = list(itertools.chain(*results_tuple))
results_2d = itertools.zip_longest(
*[results_flatten[i::n_cols] for i in range(n_cols)])
table = tabulate(
results_2d,
tablefmt='pipe',
floatfmt='.3f',
headers=['object', 'vsd', 'mssd', 'mspd', 'mean'] * (n_cols // 5),
numalign='right',
)
logger.info('Per-object scores for *{}*: \n'.format(set_str) + table)
results['per_obj'] = results_per_object
return results
