def get_gt_scene_crops(scene_id, eval_args, train_args):
dataset_name = eval_args.get("DATA", "DATASET")
cam_type = eval_args.get("DATA", "CAM_TYPE")
icp = eval_args.getboolean("EVALUATION", "ICP")
delta = eval_args.get("METRIC", "VSD_DELTA")
workspace_path = os.environ.get("AE_WORKSPACE_PATH")
dataset_path = u.get_dataset_path(workspace_path)
H = train_args.getint("Dataset", "H")
cfg_string = str([scene_id] + eval_args.items("DATA") +
eval_args.items("BBOXES") + [H])
current_config_hash = hashlib.md5(cfg_string).hexdigest()
current_file_name = os.path.join(dataset_path,
current_config_hash + ".npz")
if os.path.exists(current_file_name):
data = np.load(current_file_name)
test_img_crops = data["test_img_crops"].item()
test_img_depth_crops = data["test_img_depth_crops"].item()
bb_scores = data["bb_scores"].item()
bb_vis = data["visib_gt"].item()
bbs = data["bbs"].item()
if not osp.exists(current_file_name) or len(test_img_crops) == 0 or len(
test_img_depth_crops) == 0:
test_imgs = load_scenes(scene_id, eval_args)
test_imgs_depth = load_scenes(scene_id, eval_args,
depth=True) if icp else None
data_params = dataset_params.get_dataset_params(dataset_name,
model_type="",
train_type="",
test_type=cam_type,
cam_type=cam_type)
# only available for primesense, sixdtoolkit can generate
visib_gt = inout.load_yaml(data_params["scene_gt_stats_mpath"].format(
scene_id, delta))
bb_gt = inout.load_gt(data_params["scene_gt_mpath"].format(scene_id))
test_img_crops, test_img_depth_crops, bbs, bb_scores, bb_vis = generate_scene_crops(
test_imgs,
test_imgs_depth,
bb_gt,
eval_args,
train_args,
visib_gt=visib_gt)
np.savez(
current_file_name,
test_img_crops=test_img_crops,
test_img_depth_crops=test_img_depth_crops,
bbs=bbs,
bb_scores=bb_scores,
visib_gt=bb_vis,
)
current_cfg_file_name = os.path.join(dataset_path,
current_config_hash + ".cfg")
with open(current_cfg_file_name, "w") as f:
f.write(cfg_string)
print("created new ground truth crops!")
else:
print("loaded previously generated ground truth crops!")
print(len(test_img_crops), len(test_img_depth_crops))
return (test_img_crops, test_img_depth_crops, bbs, bb_scores, bb_vis)
def plot_vsd_err_hist(eval_dir,
scene_ids,
obj_id,
dataset_name="linemod",
top_n=1,
delta=15,
tau=20,
cost="step",
cam_type="primesense"):
# top_n_eval = eval_args.getint('EVALUATION', 'TOP_N_EVAL')
# top_n = eval_args.getint('METRIC', 'TOP_N')
# delta = eval_args.getint('METRIC', 'VSD_DELTA')
# tau = eval_args.getint('METRIC', 'VSD_TAU')
# cost = eval_args.get('METRIC', 'VSD_COST')
# cam_type = eval_args.get('DATA', 'cam_type')
# dataset_name = eval_args.get('DATA', 'dataset')
# obj_id = eval_args.getint('DATA', 'obj_id')
# if top_n_eval < 1:
# return
data_params = dataset_params.get_dataset_params(dataset_name,
model_type="",
train_type="",
test_type=cam_type,
cam_type=cam_type)
vsd_errs = []
for scene_id in scene_ids:
if dataset_name in ["linemod", "hinterstoisser"]:
# NOTE: linemod scene_id == obj_id
if obj_id != scene_id:
continue
error_file_path = osp.join(
eval_dir,
"error=vsd_ntop=%s_delta=%s_tau=%s_cost=%s" %
(top_n, delta, tau, cost),
"errors_{:02d}.yml".format(scene_id),
)
if not osp.exists(error_file_path):
print("WARNING: " + error_file_path + " not found")
continue
gts = inout.load_gt(data_params["scene_gt_mpath"].format(scene_id))
visib_gts = inout.load_yaml(data_params["scene_gt_stats_mpath"].format(
scene_id, 15)) # delta=15
vsd_dict = inout.load_yaml(error_file_path)
for view, vsd_e in enumerate(vsd_dict):
vsds = vsd_dict[view * top_n:(view + 1) * top_n]
for gt, visib_gt in zip(gts[view], visib_gts[view]):
if gt["obj_id"] == obj_id:
if visib_gt["visib_fract"] > 0.1:
for vsd_e in vsds:
vsd_errs += [list(vsd_e["errors"].values())[0]]
if len(vsd_errs) == 0:
return
vsd_errs = np.array(vsd_errs)
logger.info("vsd errs: {}".format(len(vsd_errs)))
fig = plt.figure() # noqa
ax = plt.gca()
ax.set_xlim((0.0, 1.0))
plt.grid()
plt.xlabel("vsd err")
plt.ylabel("recall")
plt.title("obj: {}, VSD Error vs Recall".format(obj_id))
legend = []
for n in np.unique(np.array([top_n, 1])):
total_views = int(len(vsd_errs) / top_n)
min_vsd_errs = np.empty((total_views, ))
for view in range(total_views):
top_n_errors = vsd_errs[view * top_n:(view + 1) * top_n]
if n == 1:
top_n_errors = top_n_errors[np.newaxis, 0]
min_vsd_errs[view] = np.min(top_n_errors)
min_vsd_errs_sorted = np.sort(min_vsd_errs)
recall = np.float32(np.arange(total_views) + 1.0) / total_views
# fill curve
min_vsd_errs_sorted = np.hstack((min_vsd_errs_sorted, np.array([1.0])))
recall = np.hstack((recall, np.array([1.0])))
AUC_vsd = np.trapz(recall, min_vsd_errs_sorted)
plt.plot(min_vsd_errs_sorted, recall)
legend += ["top {0} vsd err, AUC = {1:.4f}".format(n, AUC_vsd)]
logger.info("obj:{} top {} vsd err, AUC = {:.4f}".format(
obj_id, n, AUC_vsd))
plt.legend(legend)
out_file = osp.join(eval_dir, "latex",
"vsd_err_hist_obj_{:02d}.tex".format(obj_id))
mkdir_p(osp.dirname(out_file))
logger.info(osp.basename(out_file))
tikz_save(out_file,
figurewidth="0.45\\textheight",
figureheight="0.45\\textheight",
show_info=False)
def plot_scene_with_3DBoxes(scene_res_dirs,
dataset_name="tless",
scene_id=1,
save=False):
# sixd_img_path = '/home_local/sund_ma/data/linemod_dataset/test'
# model_path = '/home_local/sund_ma/data/linemod_dataset/models'
# inout.save_results_sixd17(res_path, preds, run_time=run_time)
# obj_gts = []
# obj_infos = []
# for object_id in range(1,noofobjects+1):
# obj_gts.append(inout.load_gt(os.path.join(sixd_img_path,'{:02d}'.format(object_id),'gt.yml')))
# obj_infos.append(inout.load_info(os.path.join(sixd_img_path,'{:02d}'.format(object_id),'info.yml')))
# print( len(obj_gts))
# dataset_name = eval_args.get('DATA','DATASET')
# cam_type = eval_args.get('DATA','CAM_TYPE')
# data_params = dataset_params.get_dataset_params(dataset_name, model_type='', train_type='', test_type='primesense', cam_type='primesense')
data_params = dataset_params.get_dataset_params(dataset_name,
model_type="",
train_type="")
models_cad_files = sorted(
glob.glob(
os.path.join(os.path.dirname(data_params["model_mpath"]),
"*.ply")))
W, H = data_params["test_im_size"]
from lib.meshrenderer import box3d_renderer
renderer_line = box3d_renderer.Renderer(models_cad_files, 1, W, H)
scene_result_dirs = sorted(glob.glob(scene_res_dirs))
print(data_params["test_rgb_mpath"])
print(data_params["scene_gt_mpath"])
# for scene_id in range(1,21):
# sixd_img_path = data_params['test_rgb_mpath'].format(scene_id)
scene_gts = inout.load_gt(data_params["scene_gt_mpath"].format(scene_id))
scene_infos = inout.load_info(
data_params["scene_info_mpath"].format(scene_id))
scene_dirs = [
d for d in scene_result_dirs if "%02d" % scene_id == d.split("/")[-1]
]
print(scene_dirs)
for view in range(len(scene_infos)):
sixd_img_path = data_params["test_rgb_mpath"].format(scene_id, view)
img = cv2.imread(sixd_img_path)
box_img = img.copy()
# cv2.imshow('',img)
# cv2.waitKey(0)
K = scene_infos[view]["cam_K"]
for bb in scene_gts[view]:
xmin = int(bb["obj_bb"][0])
ymin = int(bb["obj_bb"][1])
xmax = int(bb["obj_bb"][0] + bb["obj_bb"][2])
ymax = int(bb["obj_bb"][1] + bb["obj_bb"][3])
cv2.rectangle(box_img, (xmin, ymin), (xmax, ymax), (0, 255, 0), 2)
cv2.putText(box_img, "%s" % (bb["obj_id"]), (xmin, ymax + 20),
cv2.FONT_ITALIC, 0.5, (0, 255, 0), 2)
# for gt in scene_gts[view]:
# if gt['obj_id'] not in [1,8,9]:
# lines_gt = renderer_line.render(gt['obj_id']-1,K,gt['cam_R_m2c'],gt['cam_t_m2c'],10,5000)
# lines_gt_mask = (np.sum(lines_gt,axis=2) < 20)[:,:,None]
# print( lines_gt.shape)
# lines_gt = lines_gt[:,:,[1,0,2]]
# img = lines_gt_mask*img + lines_gt
for scene_dir in scene_dirs:
try:
res_path = glob.glob(
os.path.join(scene_dir, "%04d_*.yml" % view))
print(res_path)
res_path = res_path[0]
# print( 'here', res_path)
obj_id = int(res_path.split("_")[-1].split(".")[0])
results = inout.load_results_sixd17(res_path)
print(results)
e = results["ests"][0]
R_est = e["R"]
t_est = e["t"]
K = scene_infos[view]["cam_K"]
lines = renderer_line.render(obj_id - 1, K, R_est, t_est, 10,
5000)
lines_mask = (np.sum(lines, axis=2) < 20)[:, :, None]
# img[lines>0] = lines[lines>0]
if obj_id % 7 == 1:
lines[:, :, 0] = lines[:, :, 1]
elif obj_id % 7 == 2:
lines[:, :, 2] = lines[:, :, 1]
elif obj_id % 7 == 3:
lines[:, :, 0] = lines[:, :, 1]
lines[:, :, 1] = lines[:, :, 2]
img = lines_mask * img + lines
except:
print("undeteceted obj: ", scene_dir)
cv2.imshow("", img)
if cv2.waitKey(1) == 32:
cv2.waitKey(0)
if save:
if "icp" in scene_res_dirs:
if not os.path.exists("%02d" % scene_id):
os.makedirs("%02d" % scene_id)
cv2.imwrite(os.path.join("%02d" % scene_id, "%04d.png" % view),
img)
else:
if not os.path.exists("%02d_rgb" % scene_id):
os.makedirs("%02d_rgb" % scene_id)
cv2.imwrite(
os.path.join("%02d_rgb" % scene_id, "%04d.png" % view),
img)
def plot_R_err_hist(eval_args, eval_dir, scene_ids):
top_n_eval = eval_args.getint("EVALUATION", "TOP_N_EVAL")
top_n = eval_args.getint("METRIC", "TOP_N")
cam_type = eval_args.get("DATA", "cam_type")
dataset_name = eval_args.get("DATA", "dataset")
obj_id = eval_args.getint("DATA", "obj_id")
if top_n_eval < 1:
return
data_params = dataset_params.get_dataset_params(dataset_name,
model_type="",
train_type="",
test_type=cam_type,
cam_type=cam_type)
angle_errs = []
for scene_id in scene_ids:
error_file_path = os.path.join(eval_dir, "error=re_ntop=%s" % top_n,
"errors_{:02d}.yml".format(scene_id))
if not os.path.exists(error_file_path):
print("WARNING: " + error_file_path + " not found")
continue
# angle_errs_dict = inout.load_yaml(error_file_path)
# angle_errs += [list(angle_e['errors'].values())[0] for angle_e in angle_errs_dict]
gts = inout.load_gt(data_params["scene_gt_mpath"].format(scene_id))
visib_gts = inout.load_yaml(data_params["scene_gt_stats_mpath"].format(
scene_id, 15))
re_dict = inout.load_yaml(error_file_path)
for view in range(len(gts)):
res = re_dict[view * top_n:(view + 1) * top_n]
for gt, visib_gt in zip(gts[view], visib_gts[view]):
if gt["obj_id"] == obj_id:
if visib_gt["visib_fract"] > 0.1:
for re_e in res:
angle_errs += [list(re_e["errors"].values())[0]]
if len(angle_errs) == 0:
return
angle_errs = np.array(angle_errs)
fig = plt.figure() # noqa
plt.grid()
plt.xlabel("angle err [deg]")
plt.ylabel("recall")
plt.title("Angle Error vs Recall")
legend = []
for n in np.unique(np.array([top_n, 1])):
total_views = len(angle_errs) / top_n
min_angle_errs = np.empty((total_views, ))
min_angle_errs_rect = np.empty((total_views, ))
for view in range(total_views):
top_n_errors = angle_errs[view * top_n:(view + 1) * top_n]
if n == 1:
top_n_errors = top_n_errors[np.newaxis, 0]
min_angle_errs[view] = np.min(top_n_errors)
min_angle_errs_rect[view] = np.min(
np.hstack((top_n_errors, 180 - top_n_errors)))
min_angle_errs_sorted = np.sort(min_angle_errs)
min_angle_errs_rect_sorted = np.sort(min_angle_errs_rect)
recall = (np.arange(total_views) + 1.0) / total_views
# fill curve
min_angle_errs_sorted = np.hstack(
(min_angle_errs_sorted, np.array([180.0])))
min_angle_errs_rect_sorted = np.hstack(
(min_angle_errs_rect_sorted, np.array([90.0])))
recall = np.hstack((recall, np.array([1.0])))
AUC_angle = np.trapz(recall, min_angle_errs_sorted / 180.0)
AUC_angle_rect = np.trapz(recall, min_angle_errs_rect_sorted / 90.0)
plt.plot(min_angle_errs_sorted, recall)
plt.plot(min_angle_errs_rect_sorted, recall)
legend += [
"top {0} angle err, AUC = {1:.4f}".format(n, AUC_angle),
"top {0} rectified angle err, AUC = {1:.4f}".format(
n, AUC_angle_rect),
]
plt.legend(legend)
tikz_save(
osp.join(eval_dir, "latex", "R_err_hist.tex"),
figurewidth="0.45\\textheight",
figureheight="0.45\\textheight",
show_info=False,
)
def match_and_eval_performance_scores(
eval_dir,
error_types=["vsd", "re", "te"],
error_thresh={
"vsd": 0.3,
"cou": 0.5,
"te": 5.0,
"re": 5.0
}, # cm # deg
error_thresh_fact={
"add": 0.1,
"adi": 0.1
},
dataset="linemod",
cam_type="primesense",
n_top=1,
vsd_delta=15,
vsd_tau=20,
vsd_cost="step",
method="",
image_subset="bb8",
):
"""
error_thresh_fact: Factor k; threshold of correctness = k * d, where d is the object diameter
"""
# Paths to pose errors (calculated using eval_calc_errors.py)
# ---------------------------------------------------------------------------
# error_bpath = '/path/to/eval/'
# error_paths = [
# pjoin(error_bpath, 'hodan-iros15_hinterstoisser'),
# # pjoin(error_bpath, 'hodan-iros15_tless_primesense'),
# ]
assert image_subset in ["sixd_v1", "bb8", "None"]
test_type = cam_type
idx_th = 0
idx_thf = 0
for error_type in error_types:
# Error signature
error_sign = "error:" + error_type + "_ntop:" + str(n_top)
if error_type == "vsd":
error_sign += "_delta:{}_tau:{}_cost:{}".format(
vsd_delta, vsd_tau, vsd_cost)
error_path = osp.join(eval_dir, error_sign)
# error_dir = 'error=vsd_ntop=1_delta=15_tau=20_cost=step'
# Other paths
# ---------------------------------------------------------------------------
# Mask of path to the input file with calculated errors
errors_mpath = pjoin("{error_path}", "errors_{scene_id:02d}.yml")
# Mask of path to the output file with established matches and calculated scores
matches_mpath = pjoin("{error_path}", "matches_{eval_sign}.yml")
scores_mpath = pjoin("{error_path}", "scores_{eval_sign}.yml")
# Parameters
# ---------------------------------------------------------------------------
# use_image_subset = True # Whether to use the specified subset of images
require_all_errors = False # Whether to break if some errors are missing
visib_gt_min = 0.1 # Minimum visible surface fraction of valid GT pose
visib_delta = 15 # [mm]
# # Threshold of correctness
# error_thresh = {
# 'vsd': 0.3,
# 'cou': 0.5,
# 'te': 5.0, # [cm]
# 're': 5.0 # [deg]
# }
# # Factor k; threshold of correctness = k * d, where d is the object diameter
# error_thresh_fact = {
# 'add': 0.1,
# 'adi': 0.1
# }
# Evaluation
# ---------------------------------------------------------------------------
# Evaluation signature
if error_type in ["add", "adi"]:
if type(error_thresh_fact[error_type]) is list:
cur_thres_f = error_thresh_fact[error_type][idx_thf]
idx_thf += 1
else:
cur_thres_f = error_thresh_fact[error_type]
eval_sign = "thf:" + str(cur_thres_f)
else:
if type(error_thresh[error_type]) is list:
cur_thres = error_thresh[error_type][idx_th]
idx_th += 1
else:
cur_thres = error_thresh[error_type]
eval_sign = "th:" + str(cur_thres)
eval_sign += "_min-visib:" + str(visib_gt_min)
logger.info("--- Processing: {}, {}, {}".format(
method, dataset, error_type))
# Load dataset parameters
dp = get_dataset_params(dataset, test_type=test_type)
obj_ids = range(1, dp["obj_count"] + 1)
scene_ids = range(1, dp["scene_count"] + 1)
# Subset of images to be considered
if image_subset == "sixd_v1":
logger.info("use image subset: {}".format(dp["test_set_fpath"]))
im_ids_sets = inout.load_yaml(dp["test_set_fpath"])
total_imgs = sum([len(v) for k, v in im_ids_sets.items()])
logger.info(
"total number of imgs in test set: {}".format(total_imgs))
elif image_subset == "bb8":
logger.info("use image subset: {}".format(
dp["test_set_bb8_fpath"]))
im_ids_sets = inout.load_yaml(dp["test_set_bb8_fpath"])
total_imgs = sum([len(v) for k, v in im_ids_sets.items()])
logger.info("total number of imgs in test set: {}".format(
total_imgs)) # 13425
else:
im_ids_sets = None
# Set threshold of correctness (might be different for each object)
error_threshs = {}
if error_type in ["add", "adi"]:
# Relative to object diameter
models_info = inout.load_yaml(dp["models_info_path"])
for obj_id in obj_ids:
obj_diameter = models_info[obj_id]["diameter"]
error_threshs[obj_id] = cur_thres_f * obj_diameter
else:
# The same threshold for all objects
for obj_id in obj_ids:
error_threshs[obj_id] = cur_thres
# Go through the test scenes and match estimated poses to GT poses
# -----------------------------------------------------------------------
matches = [] # Stores info about the matching estimate for each GT
for scene_id in scene_ids:
# Load GT poses
gts = inout.load_gt(dp["scene_gt_mpath"].format(scene_id))
# Load statistics (e.g. visibility fraction) of the GT poses
gt_stats_path = dp["scene_gt_stats_mpath"].format(
scene_id, visib_delta)
gt_stats = inout.load_yaml(gt_stats_path)
# Keep the GT poses and their stats only for the selected images
if im_ids_sets is not None:
if scene_id not in im_ids_sets.keys():
continue
im_ids = im_ids_sets[scene_id]
gts = {im_id: gts[im_id] for im_id in im_ids}
gt_stats = {im_id: gt_stats[im_id] for im_id in im_ids}
# Load pre-calculated errors of the pose estimates
scene_errs_path = errors_mpath.format(error_path=error_path,
scene_id=scene_id)
if osp.isfile(scene_errs_path):
logger.info("loading error file: {}".format(scene_errs_path))
errs = inout.load_errors(scene_errs_path)
matches += match_poses(gts, gt_stats, errs, scene_id,
visib_gt_min, error_threshs, n_top)
elif require_all_errors:
raise IOError(
"{} is missing, but errors for all scenes are required"
" (require_all_results = True).".format(scene_errs_path))
# Calculate the performance scores
# -----------------------------------------------------------------------
# Split the dataset of Hinterstoisser to the original LINEMOD dataset
# and the Occlusion dataset by TUD (i.e. the extended GT for scene #2)
if dataset in ["hinterstoisser", "linemod", "linemod_occ"]:
if dataset in ["linemod", "hinterstoisser"]:
logger.info("-- LINEMOD dataset")
eval_sign_lm = "linemod_" + eval_sign
matches_lm = [
m for m in matches if m["scene_id"] == m["obj_id"]
]
scores_lm = calc_scores(scene_ids,
obj_ids,
matches_lm,
n_top,
dataset=dataset)
# Save scores
scores_lm_path = scores_mpath.format(error_path=error_path,
eval_sign=eval_sign_lm)
inout.save_yaml(scores_lm_path, scores_lm)
# Save matches
matches_path = matches_mpath.format(error_path=error_path,
eval_sign=eval_sign_lm)
inout.save_yaml(matches_path, matches_lm)
elif dataset == "linemod_occ":
logger.info("-- Occlusion dataset")
eval_sign_occ = "occlusion_" + eval_sign
matches_occ = [m for m in matches if m["scene_id"] == 2]
scene_ids_occ = [2]
# obj_ids_occ = [1, 2, 5, 6, 8, 9, 10, 11, 12]
obj_ids_occ = [1, 5, 6, 8, 9, 10, 11, 12]
scores_occ = calc_scores(scene_ids_occ,
obj_ids_occ,
matches_occ,
n_top,
dataset=dataset)
# Save scores
scores_occ_path = scores_mpath.format(error_path=error_path,
eval_sign=eval_sign_occ)
inout.save_yaml(scores_occ_path, scores_occ)
# Save matches
matches_path = matches_mpath.format(error_path=error_path,
eval_sign=eval_sign_occ)
inout.save_yaml(matches_path, matches_occ)
else:
scores = calc_scores(scene_ids, obj_ids, matches, n_top)
# Save scores
scores_path = scores_mpath.format(error_path=error_path,
eval_sign=eval_sign)
inout.save_yaml(scores_path, scores)
# Save matches
matches_path = matches_mpath.format(error_path=error_path,
eval_sign=eval_sign)
inout.save_yaml(matches_path, matches)
logger.info("Done.")