else: # 'test'
data_ids = range(1, dp['scene_count'] + 1)
gt_mpath_key = 'scene_gt_mpath'
gt_stats_mpath_key = 'scene_gt_stats_mpath'
# Subset of images to be considered
if dataset_part == 'test' and use_image_subset:
im_ids_sets = inout.load_yaml(dp['test_set_fpath'])
else:
im_ids_sets = None
# Load the GT statistics
gt_stats = []
for data_id in data_ids:
print('Loading GT stats: {}, {}'.format(dataset, data_id))
gts = inout.load_gt(dp[gt_mpath_key].format(data_id))
gt_stats_curr = inout.load_yaml(dp[gt_stats_mpath_key].format(
data_id, delta))
# Considered subset of images for the current scene
if im_ids_sets is not None:
im_ids_curr = im_ids_sets[data_id]
else:
im_ids_curr = sorted(gt_stats_curr.keys())
for im_id in im_ids_curr:
gt_stats_im = gt_stats_curr[im_id]
for gt_id, p in enumerate(gt_stats_im):
p['data_id'] = data_id
p['im_id'] = im_id
p['gt_id'] = gt_id
obj_id_in_scene_array = list()
obj_id_in_scene_array.append(scene_id)
if dataset =='doumanoglou' and scene_id == 3:
obj_id_in_scene_array = [1, 2]
if dataset == 'hinterstoisser' and scene_id == 2:
obj_id_in_scene_array = [1, 2, 5, 6, 8, 9, 10, 11, 12] # for occ dataset
for obj_id_in_scene in obj_id_in_scene_array:
# Load scene info and gt poses
print('#' * 20)
print('\nreading detector template & info, obj: {}'.format(obj_id_in_scene))
misc.ensure_dir(join(result_base_path, '{:02d}'.format(scene_id)))
scene_info = inout.load_info(dp['scene_info_mpath'].format(scene_id))
scene_gt = inout.load_gt(dp['scene_gt_mpath'].format(scene_id))
model = inout.load_ply(dp['model_mpath'].format(obj_id_in_scene))
######################################################
# prepare renderer rather than rebuilding every time
clip_near = 10 # [mm]
clip_far = 10000 # [mm]
ambient_weight = 0.8
surf_color = None
mode = 'rgb+depth'
K = dp['cam']['K']
shading = 'phong'
# Load model texture
def main():
# Paths to pose errors (calculated using eval_calc_errors.py)
#---------------------------------------------------------------------------
top_level_path = os.path.dirname(os.path.dirname(
os.path.abspath(__file__)))
dataset = 'hinterstoisser'
# dataset = 'tless'
# dataset = 'tudlight'
# dataset = 'rutgers'
# dataset = 'tejani'
# dataset = 'doumanoglou'
# dataset = 'toyotalight'
error_bpath = pjoin(top_level_path, 'eval')
error_paths = [
pjoin(error_bpath, 'patch-linemod_' + dataset),
# pjoin(error_bpath, 'hodan-iros15_tless_primesense'),
]
error_dir = 'error=vsd_ntop=1_delta=15_tau=20_cost=step'
for i in range(len(error_paths)):
error_paths[i] = os.path.join(error_paths[i], error_dir)
# Other paths
#---------------------------------------------------------------------------
# Mask of path to the input file with calculated errors
errors_mpath = pjoin('{error_path}', 'errors_{scene_id:02d}.yml')
# Mask of path to the output file with established matches and calculated scores
matches_mpath = pjoin('{error_path}', 'matches_{eval_sign}.yml')
scores_mpath = pjoin('{error_path}', 'scores_{eval_sign}.yml')
# Parameters
#---------------------------------------------------------------------------
use_image_subset = True # Whether to use the specified subset of images
require_all_errors = True # Whether to break if some errors are missing
visib_gt_min = 0.1 # Minimum visible surface fraction of valid GT pose
visib_delta = 15 # [mm]
# Threshold of correctness
error_thresh = {
'vsd': 0.3,
'cou': 0.5,
'te': 5.0, # [cm]
're': 5.0 # [deg]
}
# Factor k; threshold of correctness = k * d, where d is the object diameter
error_thresh_fact = {'add': 0.1, 'adi': 0.1}
# Evaluation
#---------------------------------------------------------------------------
for error_path in error_paths:
# Parse info about the errors from the folder names
error_sign = os.path.basename(error_path)
error_type = error_sign.split('_')[0].split('=')[1]
n_top = int(error_sign.split('_')[1].split('=')[1])
res_sign = os.path.basename(os.path.dirname(error_path)).split('_')
method = res_sign[0]
dataset = res_sign[1]
test_type = res_sign[2] if len(res_sign) > 3 else ''
# Evaluation signature
if error_type in ['add', 'adi']:
eval_sign = 'thf=' + str(error_thresh_fact[error_type])
else:
eval_sign = 'th=' + str(error_thresh[error_type])
eval_sign += '_min-visib=' + str(visib_gt_min)
print('--- Processing: {}, {}, {}'.format(method, dataset, error_type))
# Load dataset parameters
dp = get_dataset_params(dataset, test_type=test_type)
obj_ids = range(1, dp['obj_count'] + 1)
scene_ids = range(1, dp['scene_count'] + 1)
# Subset of images to be considered
if use_image_subset:
im_ids_sets = inout.load_yaml(dp['test_set_fpath'])
else:
im_ids_sets = None
# Set threshold of correctness (might be different for each object)
error_threshs = {}
if error_type in ['add', 'adi']:
# Relative to object diameter
models_info = inout.load_yaml(dp['models_info_path'])
for obj_id in obj_ids:
obj_diameter = models_info[obj_id]['diameter']
error_threshs[obj_id] = error_thresh_fact[error_type] *\
obj_diameter
else:
# The same threshold for all objects
for obj_id in obj_ids:
error_threshs[obj_id] = error_thresh[error_type]
# Go through the test scenes and match estimated poses to GT poses
#-----------------------------------------------------------------------
matches = [] # Stores info about the matching estimate for each GT
for scene_id in scene_ids:
# Load GT poses
gts = inout.load_gt(dp['scene_gt_mpath'].format(scene_id))
# Load statistics (e.g. visibility fraction) of the GT poses
gt_stats_path = dp['scene_gt_stats_mpath'].format(
scene_id, visib_delta)
gt_stats = inout.load_yaml(gt_stats_path)
# Keep the GT poses and their stats only for the selected images
if im_ids_sets is not None:
im_ids = im_ids_sets[scene_id]
gts = {im_id: gts[im_id] for im_id in im_ids}
gt_stats = {im_id: gt_stats[im_id] for im_id in im_ids}
# Load pre-calculated errors of the pose estimates
scene_errs_path = errors_mpath.format(error_path=error_path,
scene_id=scene_id)
if os.path.isfile(scene_errs_path):
errs = inout.load_errors(scene_errs_path)
matches += match_poses(gts, gt_stats, errs, scene_id,
visib_gt_min, error_threshs, n_top)
elif require_all_errors:
raise IOError(
'{} is missing, but errors for all scenes are required'
' (require_all_results = True).'.format(scene_errs_path))
# Calculate the performance scores
#-----------------------------------------------------------------------
# Split the dataset of Hinterstoisser to the original LINEMOD dataset
# and the Occlusion dataset by TUD (i.e. the extended GT for scene #2)
if dataset == 'hinterstoisser':
print('-- LINEMOD dataset')
eval_sign_lm = 'linemod_' + eval_sign
matches_lm = [m for m in matches if m['scene_id'] == m['obj_id']]
scores_lm = calc_scores(scene_ids, obj_ids, matches_lm, n_top)
# Save scores
scores_lm_path = scores_mpath.format(error_path=error_path,
eval_sign=eval_sign_lm)
inout.save_yaml(scores_lm_path, scores_lm)
# Save matches
matches_path = matches_mpath.format(error_path=error_path,
eval_sign=eval_sign_lm)
inout.save_yaml(matches_path, matches_lm)
print('-- Occlusion dataset')
eval_sign_occ = 'occlusion_' + eval_sign
matches_occ = [m for m in matches if m['scene_id'] == 2]
scene_ids_occ = [2]
obj_ids_occ = [1, 2, 5, 6, 8, 9, 10, 11, 12]
scores_occ = calc_scores(scene_ids_occ, obj_ids_occ, matches_occ,
n_top)
# Save scores
scores_occ_path = scores_mpath.format(error_path=error_path,
eval_sign=eval_sign_occ)
inout.save_yaml(scores_occ_path, scores_occ)
# Save matches
matches_path = matches_mpath.format(error_path=error_path,
eval_sign=eval_sign_occ)
inout.save_yaml(matches_path, matches_occ)
else:
scores = calc_scores(scene_ids, obj_ids, matches, n_top)
# Save scores
scores_path = scores_mpath.format(error_path=error_path,
eval_sign=eval_sign)
inout.save_yaml(scores_path, scores)
# Save matches
matches_path = matches_mpath.format(error_path=error_path,
eval_sign=eval_sign)
inout.save_yaml(matches_path, matches)
print('Done.')
if error_type in ['vsd', 'add', 'adi', 'cou']:
print('Loading object models...')
models = {}
for obj_id in range(1, dp['obj_count'] + 1):
models[obj_id] = inout.load_ply(dp['model_mpath'].format(obj_id))
# Directories with results for individual scenes
scene_dirs = sorted([d for d in glob.glob(os.path.join(result_path, '*'))
if os.path.isdir(d)])
for scene_dir in scene_dirs:
scene_id = int(os.path.basename(scene_dir))
# Load info and GT poses for the current scene
scene_info = inout.load_info(dp['scene_info_mpath'].format(scene_id))
scene_gt = inout.load_gt(dp['scene_gt_mpath'].format(scene_id))
res_paths = sorted(glob.glob(os.path.join(scene_dir, '*.yml')))
errs = []
im_id = -1
depth_im = None
for res_id, res_path in enumerate(res_paths):
# t = time.time()
# Parse image ID and object ID from the filename
filename = os.path.basename(res_path).split('.')[0]
im_id_prev = im_id
im_id, obj_id = map(int, filename.split('_'))
if res_id % 10 == 0:
dataset_str = dataset
def load_scene(scene_id):
"""
Loads the specified scene.
"""
clean()
global ref_im_ind
global scene_info
global scene_gt
# Load scene info
scene_info_path = par['scene_info_mpath'].format(scene_id)
print('Loading scene info: ' + scene_info_path)
scene_info = inout.load_info(scene_info_path)
ref_im_id = sorted(scene_info.keys())[ref_im_ind]
scene_info_ref = scene_info[ref_im_id]
R_w2c = scene_info_ref['cam_R_w2c']
R_w2c_inv = np.linalg.inv(R_w2c)
t_w2c = scene_info_ref['cam_t_w2c']
# Load ground truth poses for the reference camera coordinate system
scene_gt_path = par['scene_gt_mpath'].format(scene_id)
print('Loading GT poses: ' + scene_gt_path)
scene_gt = inout.load_gt(scene_gt_path)
scene_gt_ref = scene_gt[ref_im_id]
# Load scene model
#bpy.ops.import_mesh.ply(filepath=scene_model_mpath.format(scene_id))
# Load models of objects that are present in the scene
for gt in scene_gt_ref:
model_path = par['model_mpath'].format(gt['obj_id'])
print('Loading model {}: {}'.format(gt['obj_id'], model_path))
bpy.ops.import_mesh.ply(filepath=model_path)
# Take poses from the reference camera coordinate system and transform them
# to the world coordinate system (using the known camera-to-world trans.)
objs = list(bpy.data.objects)
objs_moved = [False for _ in objs]
for gt in scene_gt_ref:
for i, obj in enumerate(objs):
obj_id = int(obj.name.split('obj_')[1].split('.')[0])
if gt['obj_id'] == obj_id and not objs_moved[i]:
print('Setting pose of model {}'.format(gt['obj_id']))
#print(i, gt, obj_id, objs_moved[i], obj.location)
R_m2w = R_w2c_inv.dot(gt['cam_R_m2c'])
t_m2w = R_w2c_inv.dot(gt['cam_t_m2c']) - R_w2c_inv.dot(t_w2c)
obj.rotation_euler = transform.euler_from_matrix(R_m2w, 'sxyz')
obj.location = t_m2w.flatten().tolist()
objs_moved[i] = True
break
# Align view to the loaded elements
for area in bpy.context.screen.areas:
if area.type == 'VIEW_3D':
ctx = bpy.context.copy()
ctx['area'] = area
ctx['region'] = area.regions[-1]
bpy.ops.view3d.view_selected(ctx) # points view
# bpy.ops.view3d.camera_to_view_selected(ctx) # points camera
print("Scene loaded.")
# 'd}_{:02d}_diff_below_delta={}.jpg'
print('Loading object models...')
models = {}
for obj_id in obj_ids:
models[obj_id] = inout.load_ply(dp['model_mpath'].format(obj_id))
# visib_to_below_delta_fracs = []
for data_id in data_ids:
if do_vis:
misc.ensure_dir(os.path.dirname(
vis_mpath.format(dataset, delta, data_id, 0, 0)))
# Load scene info and gts
info = inout.load_info(dp[info_mpath_key].format(data_id))
gts = inout.load_gt(dp[gt_mpath_key].format(data_id))
# Considered subset of images for the current scene
if im_ids_sets is not None:
im_ids = im_ids_sets[data_id]
else:
im_ids = sorted(gts.keys())
gt_stats = {}
for im_id in im_ids:
print('dataset: {}, scene/obj: {}, im: {}'.format(dataset, data_id, im_id))
K = info[im_id]['cam_K']
depth_path = dp[depth_mpath_key].format(data_id, im_id)
depth_im = inout.load_depth(depth_path)
depth_im *= dp['cam']['depth_scale'] # to [mm]
def main():
# Paths to pose errors (calculated using eval_calc_errors.py)
# ---------------------------------------------------------------------------
error_bpath = "/path/to/eval/"
error_paths = [
pjoin(error_bpath, "hodan-iros15_hinterstoisser"),
# pjoin(error_bpath, 'hodan-iros15_tless_primesense'),
]
error_dir = "error:vsd_ntop:1_delta:15_tau:20_cost:step"
for i in range(len(error_paths)):
error_paths[i] = os.path.join(error_paths[i], error_dir)
# Other paths
# ---------------------------------------------------------------------------
# Mask of path to the input file with calculated errors
errors_mpath = pjoin("{error_path}", "errors_{scene_id:02d}.yml")
# Mask of path to the output file with established matches and calculated scores
matches_mpath = pjoin("{error_path}", "matches_{eval_sign}.yml")
scores_mpath = pjoin("{error_path}", "scores_{eval_sign}.yml")
# Parameters
# ---------------------------------------------------------------------------
use_image_subset = True # Whether to use the specified subset of images
require_all_errors = True # Whether to break if some errors are missing
visib_gt_min = 0.1 # Minimum visible surface fraction of valid GT pose
visib_delta = 15 # [mm]
# Threshold of correctness
error_thresh = {
"vsd": 0.3,
"cou": 0.5,
"te": 5.0,
"re": 5.0
} # [cm] # [deg]
# Factor k; threshold of correctness = k * d, where d is the object diameter
error_thresh_fact = {"add": 0.1, "adi": 0.1}
# Evaluation
# ---------------------------------------------------------------------------
for error_path in error_paths:
# Parse info about the errors from the folder names
error_sign = os.path.basename(error_path)
error_type = error_sign.split("_")[0].split(":")[1]
n_top = int(error_sign.split("_")[1].split(":")[1])
res_sign = os.path.basename(os.path.dirname(error_path)).split("_")
method = res_sign[0]
dataset = res_sign[1]
test_type = res_sign[2] if len(res_sign) > 3 else ""
# Evaluation signature
if error_type in ["add", "adi"]:
eval_sign = "thf:" + str(error_thresh_fact[error_type])
else:
eval_sign = "th:" + str(error_thresh[error_type])
eval_sign += "_min-visib:" + str(visib_gt_min)
print("--- Processing: {}, {}, {}".format(method, dataset, error_type))
# Load dataset parameters
dp = get_dataset_params(dataset, test_type=test_type)
obj_ids = range(1, dp["obj_count"] + 1)
scene_ids = range(1, dp["scene_count"] + 1)
# Subset of images to be considered
if use_image_subset:
im_ids_sets = inout.load_yaml(dp["test_set_fpath"])
else:
im_ids_sets = None
# Set threshold of correctness (might be different for each object)
error_threshs = {}
if error_type in ["add", "adi"]:
# Relative to object diameter
models_info = inout.load_yaml(dp["models_info_path"])
for obj_id in obj_ids:
obj_diameter = models_info[obj_id]["diameter"]
error_threshs[
obj_id] = error_thresh_fact[error_type] * obj_diameter
else:
# The same threshold for all objects
for obj_id in obj_ids:
error_threshs[obj_id] = error_thresh[error_type]
# Go through the test scenes and match estimated poses to GT poses
# -----------------------------------------------------------------------
matches = [] # Stores info about the matching estimate for each GT
for scene_id in scene_ids:
# Load GT poses
gts = inout.load_gt(dp["scene_gt_mpath"].format(scene_id))
# Load statistics (e.g. visibility fraction) of the GT poses
gt_stats_path = dp["scene_gt_stats_mpath"].format(
scene_id, visib_delta)
gt_stats = inout.load_yaml(gt_stats_path)
# Keep the GT poses and their stats only for the selected images
if im_ids_sets is not None:
im_ids = im_ids_sets[scene_id]
gts = {im_id: gts[im_id] for im_id in im_ids}
gt_stats = {im_id: gt_stats[im_id] for im_id in im_ids}
# Load pre-calculated errors of the pose estimates
scene_errs_path = errors_mpath.format(error_path=error_path,
scene_id=scene_id)
if os.path.isfile(scene_errs_path):
errs = inout.load_errors(scene_errs_path)
matches += match_poses(gts, gt_stats, errs, scene_id,
visib_gt_min, error_threshs, n_top)
elif require_all_errors:
raise IOError(
"{} is missing, but errors for all scenes are required"
" (require_all_results = True).".format(scene_errs_path))
# Calculate the performance scores
# -----------------------------------------------------------------------
# Split the dataset of Hinterstoisser to the original LINEMOD dataset
# and the Occlusion dataset by TUD (i.e. the extended GT for scene #2)
if dataset == "hinterstoisser":
print("-- LINEMOD dataset")
eval_sign_lm = "linemod_" + eval_sign
matches_lm = [m for m in matches if m["scene_id"] == m["obj_id"]]
scores_lm = calc_scores(scene_ids, obj_ids, matches_lm, n_top)
# Save scores
scores_lm_path = scores_mpath.format(error_path=error_path,
eval_sign=eval_sign_lm)
inout.save_yaml(scores_lm_path, scores_lm)
# Save matches
matches_path = matches_mpath.format(error_path=error_path,
eval_sign=eval_sign_lm)
inout.save_yaml(matches_path, matches_lm)
print("-- Occlusion dataset")
eval_sign_occ = "occlusion_" + eval_sign
matches_occ = [m for m in matches if m["scene_id"] == 2]
scene_ids_occ = [2]
obj_ids_occ = [1, 2, 5, 6, 8, 9, 10, 11, 12]
scores_occ = calc_scores(scene_ids_occ, obj_ids_occ, matches_occ,
n_top)
# Save scores
scores_occ_path = scores_mpath.format(error_path=error_path,
eval_sign=eval_sign_occ)
inout.save_yaml(scores_occ_path, scores_occ)
# Save matches
matches_path = matches_mpath.format(error_path=error_path,
eval_sign=eval_sign_occ)
inout.save_yaml(matches_path, matches_occ)
else:
scores = calc_scores(scene_ids, obj_ids, matches, n_top)
# Save scores
scores_path = scores_mpath.format(error_path=error_path,
eval_sign=eval_sign)
inout.save_yaml(scores_path, scores)
# Save matches
matches_path = matches_mpath.format(error_path=error_path,
eval_sign=eval_sign)
inout.save_yaml(matches_path, matches)
print("Done.")
