print('Target count: {:d}'.format(tars))
print('TP count: {:d}'.format(tps))
print('Total recall: {:.4f}'.format(total_recall))
print('Mean object recall: {:.4f}'.format(mean_obj_recall))
print('Mean scene recall: {:.4f}'.format(mean_scene_recall))
# print('Object recalls: {}'.format(str(obj_recalls)))
# print('Scene recalls: {}'.format(str(scene_recalls)))
print('')
# Evaluation signature
eval_sign = ''
if error_type in ['add', 'adi']:
eval_sign = 'thf=' + str(error_thresh_fact[error_type])
else:
eval_sign = 'th=' + str(error_thresh[error_type])
# Save scores
print('Saving scores...')
scores_path = scores_mpath.format(error_path=error_path,
eval_sign=eval_sign)
inout.save_yaml(scores_path, scores)
# Save matches
print('Saving matches...')
matches_path = matches_mpath.format(error_path=error_path,
eval_sign=eval_sign)
inout.save_yaml(matches_path, matches)
print('')
print('Done.')
# provided for the SIXD Challenge 2017.
rgb = cv2.resize(rgb, par['cam']['im_size'], interpolation=cv2.INTER_AREA)
#rgb = scipy.misc.imresize(rgb, par['cam']['im_size'][::-1], 'bicubic')
# Save the rendered images
inout.save_im(out_rgb_mpath.format(obj_id, im_id), rgb)
inout.save_depth(out_depth_mpath.format(obj_id, im_id), depth)
# Get 2D bounding box of the object model at the ground truth pose
ys, xs = np.nonzero(depth > 0)
obj_bb = misc.calc_2d_bbox(xs, ys, par['cam']['im_size'])
obj_info[im_id] = {
'cam_K': par['cam']['K'].flatten().tolist(),
'view_level': int(views_level[view_id]),
#'sphere_radius': float(radius)
}
obj_gt[im_id] = [{
'cam_R_m2c': view['R'].flatten().tolist(),
'cam_t_m2c': view['t'].flatten().tolist(),
'obj_bb': [int(x) for x in obj_bb],
'obj_id': int(obj_id)
}]
im_id += 1
# Save metadata
inout.save_yaml(out_obj_info_path.format(obj_id), obj_info)
inout.save_yaml(out_obj_gt_path.format(obj_id), obj_gt)
vis = 0.5 * depth_im_vis + 0.5 * visib_gt_vis
vis[vis > 1] = 1
vis_path = vis_mpath.format(dataset, delta, data_id, im_id,
gt_id)
inout.save_im(vis_path, vis)
# Mask of depth differences below delta
# mask_below_delta_vis = np.dstack([mask_below_delta,
# zero_ch, zero_ch])
# vis_delta = 0.5 * depth_im_vis + 0.5 * mask_below_delta_vis
# vis_delta[vis_delta > 1] = 1
# vis_delta_path = vis_delta_mpath.format(
# dataset, delta, data_id, im_id, gt_id, delta)
# inout.save_im(vis_delta_path, vis_delta)
res_path = dp[gt_stats_mpath_key].format(data_id, delta)
misc.ensure_dir(os.path.dirname(res_path))
inout.save_yaml(res_path, gt_stats)
# visib_to_below_delta_fracs = sorted(visib_to_below_delta_fracs,
# key=lambda x: x['frac'], reverse=True)
# for i in range(200):
# e = visib_to_below_delta_fracs[i]
# print('{}: data_id: {}, im_id: {}, gt_id: {}, frac: {}'.format(
# i, e['data_id'], e['im_id'], e['gt_id'], e['frac']
# ))
#
# import matplotlib.pyplot as plt
# plt.plot([e['frac'] for e in visib_to_below_delta_fracs])
# plt.show()
def main():
# Paths to pose errors (calculated using eval_calc_errors.py)
#---------------------------------------------------------------------------
top_level_path = os.path.dirname(os.path.dirname(
os.path.abspath(__file__)))
dataset = 'hinterstoisser'
# dataset = 'tless'
# dataset = 'tudlight'
# dataset = 'rutgers'
# dataset = 'tejani'
# dataset = 'doumanoglou'
# dataset = 'toyotalight'
error_bpath = pjoin(top_level_path, 'eval')
error_paths = [
pjoin(error_bpath, 'patch-linemod_' + dataset),
# pjoin(error_bpath, 'hodan-iros15_tless_primesense'),
]
error_dir = 'error=vsd_ntop=1_delta=15_tau=20_cost=step'
for i in range(len(error_paths)):
error_paths[i] = os.path.join(error_paths[i], error_dir)
# Other paths
#---------------------------------------------------------------------------
# Mask of path to the input file with calculated errors
errors_mpath = pjoin('{error_path}', 'errors_{scene_id:02d}.yml')
# Mask of path to the output file with established matches and calculated scores
matches_mpath = pjoin('{error_path}', 'matches_{eval_sign}.yml')
scores_mpath = pjoin('{error_path}', 'scores_{eval_sign}.yml')
# Parameters
#---------------------------------------------------------------------------
use_image_subset = True # Whether to use the specified subset of images
require_all_errors = True # Whether to break if some errors are missing
visib_gt_min = 0.1 # Minimum visible surface fraction of valid GT pose
visib_delta = 15 # [mm]
# Threshold of correctness
error_thresh = {
'vsd': 0.3,
'cou': 0.5,
'te': 5.0, # [cm]
're': 5.0 # [deg]
}
# Factor k; threshold of correctness = k * d, where d is the object diameter
error_thresh_fact = {'add': 0.1, 'adi': 0.1}
# Evaluation
#---------------------------------------------------------------------------
for error_path in error_paths:
# Parse info about the errors from the folder names
error_sign = os.path.basename(error_path)
error_type = error_sign.split('_')[0].split('=')[1]
n_top = int(error_sign.split('_')[1].split('=')[1])
res_sign = os.path.basename(os.path.dirname(error_path)).split('_')
method = res_sign[0]
dataset = res_sign[1]
test_type = res_sign[2] if len(res_sign) > 3 else ''
# Evaluation signature
if error_type in ['add', 'adi']:
eval_sign = 'thf=' + str(error_thresh_fact[error_type])
else:
eval_sign = 'th=' + str(error_thresh[error_type])
eval_sign += '_min-visib=' + str(visib_gt_min)
print('--- Processing: {}, {}, {}'.format(method, dataset, error_type))
# Load dataset parameters
dp = get_dataset_params(dataset, test_type=test_type)
obj_ids = range(1, dp['obj_count'] + 1)
scene_ids = range(1, dp['scene_count'] + 1)
# Subset of images to be considered
if use_image_subset:
im_ids_sets = inout.load_yaml(dp['test_set_fpath'])
else:
im_ids_sets = None
# Set threshold of correctness (might be different for each object)
error_threshs = {}
if error_type in ['add', 'adi']:
# Relative to object diameter
models_info = inout.load_yaml(dp['models_info_path'])
for obj_id in obj_ids:
obj_diameter = models_info[obj_id]['diameter']
error_threshs[obj_id] = error_thresh_fact[error_type] *\
obj_diameter
else:
# The same threshold for all objects
for obj_id in obj_ids:
error_threshs[obj_id] = error_thresh[error_type]
# Go through the test scenes and match estimated poses to GT poses
#-----------------------------------------------------------------------
matches = [] # Stores info about the matching estimate for each GT
for scene_id in scene_ids:
# Load GT poses
gts = inout.load_gt(dp['scene_gt_mpath'].format(scene_id))
# Load statistics (e.g. visibility fraction) of the GT poses
gt_stats_path = dp['scene_gt_stats_mpath'].format(
scene_id, visib_delta)
gt_stats = inout.load_yaml(gt_stats_path)
# Keep the GT poses and their stats only for the selected images
if im_ids_sets is not None:
im_ids = im_ids_sets[scene_id]
gts = {im_id: gts[im_id] for im_id in im_ids}
gt_stats = {im_id: gt_stats[im_id] for im_id in im_ids}
# Load pre-calculated errors of the pose estimates
scene_errs_path = errors_mpath.format(error_path=error_path,
scene_id=scene_id)
if os.path.isfile(scene_errs_path):
errs = inout.load_errors(scene_errs_path)
matches += match_poses(gts, gt_stats, errs, scene_id,
visib_gt_min, error_threshs, n_top)
elif require_all_errors:
raise IOError(
'{} is missing, but errors for all scenes are required'
' (require_all_results = True).'.format(scene_errs_path))
# Calculate the performance scores
#-----------------------------------------------------------------------
# Split the dataset of Hinterstoisser to the original LINEMOD dataset
# and the Occlusion dataset by TUD (i.e. the extended GT for scene #2)
if dataset == 'hinterstoisser':
print('-- LINEMOD dataset')
eval_sign_lm = 'linemod_' + eval_sign
matches_lm = [m for m in matches if m['scene_id'] == m['obj_id']]
scores_lm = calc_scores(scene_ids, obj_ids, matches_lm, n_top)
# Save scores
scores_lm_path = scores_mpath.format(error_path=error_path,
eval_sign=eval_sign_lm)
inout.save_yaml(scores_lm_path, scores_lm)
# Save matches
matches_path = matches_mpath.format(error_path=error_path,
eval_sign=eval_sign_lm)
inout.save_yaml(matches_path, matches_lm)
print('-- Occlusion dataset')
eval_sign_occ = 'occlusion_' + eval_sign
matches_occ = [m for m in matches if m['scene_id'] == 2]
scene_ids_occ = [2]
obj_ids_occ = [1, 2, 5, 6, 8, 9, 10, 11, 12]
scores_occ = calc_scores(scene_ids_occ, obj_ids_occ, matches_occ,
n_top)
# Save scores
scores_occ_path = scores_mpath.format(error_path=error_path,
eval_sign=eval_sign_occ)
inout.save_yaml(scores_occ_path, scores_occ)
# Save matches
matches_path = matches_mpath.format(error_path=error_path,
eval_sign=eval_sign_occ)
inout.save_yaml(matches_path, matches_occ)
else:
scores = calc_scores(scene_ids, obj_ids, matches, n_top)
# Save scores
scores_path = scores_mpath.format(error_path=error_path,
eval_sign=eval_sign)
inout.save_yaml(scores_path, scores)
# Save matches
matches_path = matches_mpath.format(error_path=error_path,
eval_sign=eval_sign)
inout.save_yaml(matches_path, matches)
print('Done.')
vis = 0.5 * depth_im_vis + 0.5 * visib_gt_vis
vis[vis > 1] = 1
vis_path = vis_mpath.format(
dataset, delta, data_id, im_id, gt_id)
inout.save_im(vis_path, vis)
# Mask of depth differences below delta
# mask_below_delta_vis = np.dstack([mask_below_delta,
# zero_ch, zero_ch])
# vis_delta = 0.5 * depth_im_vis + 0.5 * mask_below_delta_vis
# vis_delta[vis_delta > 1] = 1
# vis_delta_path = vis_delta_mpath.format(
# dataset, delta, data_id, im_id, gt_id, delta)
# inout.save_im(vis_delta_path, vis_delta)
res_path = dp[gt_stats_mpath_key].format(data_id, delta)
misc.ensure_dir(os.path.dirname(res_path))
inout.save_yaml(res_path, gt_stats)
# visib_to_below_delta_fracs = sorted(visib_to_below_delta_fracs,
# key=lambda x: x['frac'], reverse=True)
# for i in range(200):
# e = visib_to_below_delta_fracs[i]
# print('{}: data_id: {}, im_id: {}, gt_id: {}, frac: {}'.format(
# i, e['data_id'], e['im_id'], e['gt_id'], e['frac']
# ))
#
# import matplotlib.pyplot as plt
# plt.plot([e['frac'] for e in visib_to_below_delta_fracs])
# plt.show()
rgb = cv2.resize(rgb, p['cam']['im_size'], interpolation=cv2.INTER_AREA)
# rgb = scipy.misc.imresize(rgb, par['cam']['im_size'][::-1], 'bicubic')
# Save the rendered images
inout.save_im(out_rgb_mpath.format(obj_id, im_id), rgb)
inout.save_depth(out_depth_mpath.format(obj_id, im_id), depth)
# Get 2D bounding box of the object model at the ground truth pose
ys, xs = np.nonzero(depth > 0)
obj_bb = misc.calc_2d_bbox(xs, ys, p['cam']['im_size'])
obj_info[im_id] = {
'cam_K': p['cam']['K'].flatten().tolist(),
'view_level': int(views_level[view_id]),
# 'sphere_radius': float(radius)
}
obj_gt[im_id] = [{
'cam_R_m2c': view['R'].flatten().tolist(),
'cam_t_m2c': view['t'].flatten().tolist(),
'obj_bb': [int(x) for x in obj_bb],
'obj_id': int(obj_id)
}]
im_id += 1
# Save metadata
inout.save_yaml(out_obj_info_path.format(obj_id), obj_info)
inout.save_yaml(out_obj_gt_path.format(obj_id), obj_gt)
print("end of line for break points")
def main():
# Paths to pose errors (calculated using eval_calc_errors.py)
# ---------------------------------------------------------------------------
error_bpath = "/path/to/eval/"
error_paths = [
pjoin(error_bpath, "hodan-iros15_hinterstoisser"),
# pjoin(error_bpath, 'hodan-iros15_tless_primesense'),
]
error_dir = "error:vsd_ntop:1_delta:15_tau:20_cost:step"
for i in range(len(error_paths)):
error_paths[i] = os.path.join(error_paths[i], error_dir)
# Other paths
# ---------------------------------------------------------------------------
# Mask of path to the input file with calculated errors
errors_mpath = pjoin("{error_path}", "errors_{scene_id:02d}.yml")
# Mask of path to the output file with established matches and calculated scores
matches_mpath = pjoin("{error_path}", "matches_{eval_sign}.yml")
scores_mpath = pjoin("{error_path}", "scores_{eval_sign}.yml")
# Parameters
# ---------------------------------------------------------------------------
use_image_subset = True # Whether to use the specified subset of images
require_all_errors = True # Whether to break if some errors are missing
visib_gt_min = 0.1 # Minimum visible surface fraction of valid GT pose
visib_delta = 15 # [mm]
# Threshold of correctness
error_thresh = {
"vsd": 0.3,
"cou": 0.5,
"te": 5.0,
"re": 5.0
} # [cm] # [deg]
# Factor k; threshold of correctness = k * d, where d is the object diameter
error_thresh_fact = {"add": 0.1, "adi": 0.1}
# Evaluation
# ---------------------------------------------------------------------------
for error_path in error_paths:
# Parse info about the errors from the folder names
error_sign = os.path.basename(error_path)
error_type = error_sign.split("_")[0].split(":")[1]
n_top = int(error_sign.split("_")[1].split(":")[1])
res_sign = os.path.basename(os.path.dirname(error_path)).split("_")
method = res_sign[0]
dataset = res_sign[1]
test_type = res_sign[2] if len(res_sign) > 3 else ""
# Evaluation signature
if error_type in ["add", "adi"]:
eval_sign = "thf:" + str(error_thresh_fact[error_type])
else:
eval_sign = "th:" + str(error_thresh[error_type])
eval_sign += "_min-visib:" + str(visib_gt_min)
print("--- Processing: {}, {}, {}".format(method, dataset, error_type))
# Load dataset parameters
dp = get_dataset_params(dataset, test_type=test_type)
obj_ids = range(1, dp["obj_count"] + 1)
scene_ids = range(1, dp["scene_count"] + 1)
# Subset of images to be considered
if use_image_subset:
im_ids_sets = inout.load_yaml(dp["test_set_fpath"])
else:
im_ids_sets = None
# Set threshold of correctness (might be different for each object)
error_threshs = {}
if error_type in ["add", "adi"]:
# Relative to object diameter
models_info = inout.load_yaml(dp["models_info_path"])
for obj_id in obj_ids:
obj_diameter = models_info[obj_id]["diameter"]
error_threshs[
obj_id] = error_thresh_fact[error_type] * obj_diameter
else:
# The same threshold for all objects
for obj_id in obj_ids:
error_threshs[obj_id] = error_thresh[error_type]
# Go through the test scenes and match estimated poses to GT poses
# -----------------------------------------------------------------------
matches = [] # Stores info about the matching estimate for each GT
for scene_id in scene_ids:
# Load GT poses
gts = inout.load_gt(dp["scene_gt_mpath"].format(scene_id))
# Load statistics (e.g. visibility fraction) of the GT poses
gt_stats_path = dp["scene_gt_stats_mpath"].format(
scene_id, visib_delta)
gt_stats = inout.load_yaml(gt_stats_path)
# Keep the GT poses and their stats only for the selected images
if im_ids_sets is not None:
im_ids = im_ids_sets[scene_id]
gts = {im_id: gts[im_id] for im_id in im_ids}
gt_stats = {im_id: gt_stats[im_id] for im_id in im_ids}
# Load pre-calculated errors of the pose estimates
scene_errs_path = errors_mpath.format(error_path=error_path,
scene_id=scene_id)
if os.path.isfile(scene_errs_path):
errs = inout.load_errors(scene_errs_path)
matches += match_poses(gts, gt_stats, errs, scene_id,
visib_gt_min, error_threshs, n_top)
elif require_all_errors:
raise IOError(
"{} is missing, but errors for all scenes are required"
" (require_all_results = True).".format(scene_errs_path))
# Calculate the performance scores
# -----------------------------------------------------------------------
# Split the dataset of Hinterstoisser to the original LINEMOD dataset
# and the Occlusion dataset by TUD (i.e. the extended GT for scene #2)
if dataset == "hinterstoisser":
print("-- LINEMOD dataset")
eval_sign_lm = "linemod_" + eval_sign
matches_lm = [m for m in matches if m["scene_id"] == m["obj_id"]]
scores_lm = calc_scores(scene_ids, obj_ids, matches_lm, n_top)
# Save scores
scores_lm_path = scores_mpath.format(error_path=error_path,
eval_sign=eval_sign_lm)
inout.save_yaml(scores_lm_path, scores_lm)
# Save matches
matches_path = matches_mpath.format(error_path=error_path,
eval_sign=eval_sign_lm)
inout.save_yaml(matches_path, matches_lm)
print("-- Occlusion dataset")
eval_sign_occ = "occlusion_" + eval_sign
matches_occ = [m for m in matches if m["scene_id"] == 2]
scene_ids_occ = [2]
obj_ids_occ = [1, 2, 5, 6, 8, 9, 10, 11, 12]
scores_occ = calc_scores(scene_ids_occ, obj_ids_occ, matches_occ,
n_top)
# Save scores
scores_occ_path = scores_mpath.format(error_path=error_path,
eval_sign=eval_sign_occ)
inout.save_yaml(scores_occ_path, scores_occ)
# Save matches
matches_path = matches_mpath.format(error_path=error_path,
eval_sign=eval_sign_occ)
inout.save_yaml(matches_path, matches_occ)
else:
scores = calc_scores(scene_ids, obj_ids, matches, n_top)
# Save scores
scores_path = scores_mpath.format(error_path=error_path,
eval_sign=eval_sign)
inout.save_yaml(scores_path, scores)
# Save matches
matches_path = matches_mpath.format(error_path=error_path,
eval_sign=eval_sign)
inout.save_yaml(matches_path, matches)
print("Done.")