def compute_mean_re_te(pred_transes, pred_rots, gt_transes, gt_rots):
pred_transes = pred_transes.detach().cpu().numpy()
pred_rots = pred_rots.detach().cpu().numpy()
gt_transes = gt_transes.detach().cpu().numpy()
gt_rots = gt_rots.detach().cpu().numpy()
bs = pred_rots.shape[0]
R_errs = np.zeros((bs,), dtype=np.float32)
T_errs = np.zeros((bs,), dtype=np.float32)
for i in range(bs):
R_errs[i] = re(pred_rots[i], gt_rots[i])
T_errs[i] = te(pred_transes[i], gt_transes[i])
return R_errs.mean(), T_errs.mean()
def _eval_predictions_precision(self):
"""NOTE: eval precision instead of recall
Evaluate self._predictions on 6d pose.
Return results with the metrics of the tasks.
"""
self._logger.info("Eval results ...")
cfg = self.cfg
method_name = f"{cfg.EXP_ID.replace('_', '-')}"
cache_path = osp.join(self._output_dir,
f"{method_name}_{self.dataset_name}_preds.pkl")
if osp.exists(cache_path) and self.use_cache:
self._logger.info("load cached predictions")
self._predictions = mmcv.load(cache_path)
else:
if hasattr(self, "_predictions"):
mmcv.dump(self._predictions, cache_path)
else:
raise RuntimeError("Please run inference first")
precisions = OrderedDict()
errors = OrderedDict()
self.get_gts()
error_names = ["ad", "re", "te", "proj"]
metric_names = [
"ad_2",
"ad_5",
"ad_10",
"rete_2",
"rete_5",
"rete_10",
"re_2",
"re_5",
"re_10",
"te_2",
"te_5",
"te_10",
"proj_2",
"proj_5",
"proj_10",
]
for obj_name in self.gts:
if obj_name not in self._predictions:
continue
cur_label = self.obj_names.index(obj_name)
if obj_name not in precisions:
precisions[obj_name] = OrderedDict()
for metric_name in metric_names:
precisions[obj_name][metric_name] = []
if obj_name not in errors:
errors[obj_name] = OrderedDict()
for err_name in error_names:
errors[obj_name][err_name] = []
#################
obj_gts = self.gts[obj_name]
obj_preds = self._predictions[obj_name]
for file_name, gt_anno in obj_gts.items():
# compute precision as in DPOD paper
if file_name not in obj_preds: # no pred found
# NOTE: just ignore undetected
continue
# compute each metric
R_pred = obj_preds[file_name]["R"]
t_pred = obj_preds[file_name]["t"]
R_gt = gt_anno["R"]
t_gt = gt_anno["t"]
t_error = te(t_pred, t_gt)
if obj_name in cfg.DATASETS.SYM_OBJS:
R_gt_sym = get_closest_rot(
R_pred, R_gt, self._metadata.sym_infos[cur_label])
r_error = re(R_pred, R_gt_sym)
proj_2d_error = arp_2d(
R_pred,
t_pred,
R_gt_sym,
t_gt,
pts=self.models_3d[cur_label]["pts"],
K=gt_anno["K"])
ad_error = adi(R_pred,
t_pred,
R_gt,
t_gt,
pts=self.models_3d[self.obj_names.index(
obj_name)]["pts"])
else:
r_error = re(R_pred, R_gt)
proj_2d_error = arp_2d(
R_pred,
t_pred,
R_gt,
t_gt,
pts=self.models_3d[cur_label]["pts"],
K=gt_anno["K"])
ad_error = add(R_pred,
t_pred,
R_gt,
t_gt,
pts=self.models_3d[self.obj_names.index(
obj_name)]["pts"])
#########
errors[obj_name]["ad"].append(ad_error)
errors[obj_name]["re"].append(r_error)
errors[obj_name]["te"].append(t_error)
errors[obj_name]["proj"].append(proj_2d_error)
############
precisions[obj_name]["ad_2"].append(
float(ad_error < 0.02 * self.diameters[cur_label]))
precisions[obj_name]["ad_5"].append(
float(ad_error < 0.05 * self.diameters[cur_label]))
precisions[obj_name]["ad_10"].append(
float(ad_error < 0.1 * self.diameters[cur_label]))
# deg, cm
precisions[obj_name]["rete_2"].append(
float(r_error < 2 and t_error < 0.02))
precisions[obj_name]["rete_5"].append(
float(r_error < 5 and t_error < 0.05))
precisions[obj_name]["rete_10"].append(
float(r_error < 10 and t_error < 0.1))
precisions[obj_name]["re_2"].append(float(r_error < 2))
precisions[obj_name]["re_5"].append(float(r_error < 5))
precisions[obj_name]["re_10"].append(float(r_error < 10))
precisions[obj_name]["te_2"].append(float(t_error < 0.02))
precisions[obj_name]["te_5"].append(float(t_error < 0.05))
precisions[obj_name]["te_10"].append(float(t_error < 0.1))
# px
precisions[obj_name]["proj_2"].append(float(proj_2d_error < 2))
precisions[obj_name]["proj_5"].append(float(proj_2d_error < 5))
precisions[obj_name]["proj_10"].append(
float(proj_2d_error < 10))
# summarize
obj_names = sorted(list(precisions.keys()))
header = ["objects"] + obj_names + [f"Avg({len(obj_names)})"]
big_tab = [header]
for metric_name in metric_names:
line = [metric_name]
this_line_res = []
for obj_name in obj_names:
res = precisions[obj_name][metric_name]
if len(res) > 0:
line.append(f"{100 * np.mean(res):.2f}")
this_line_res.append(np.mean(res))
else:
line.append(0.0)
this_line_res.append(0.0)
# mean
if len(obj_names) > 0:
line.append(f"{100 * np.mean(this_line_res):.2f}")
big_tab.append(line)
for error_name in ["re", "te"]:
line = [error_name]
this_line_res = []
for obj_name in obj_names:
res = errors[obj_name][error_name]
if len(res) > 0:
line.append(f"{np.mean(res):.2f}")
this_line_res.append(np.mean(res))
else:
line.append(float("nan"))
this_line_res.append(float("nan"))
# mean
if len(obj_names) > 0:
line.append(f"{np.mean(this_line_res):.2f}")
big_tab.append(line)
### log big table
self._logger.info("precisions")
res_log_tab_str = tabulate(
big_tab,
tablefmt="plain",
# floatfmt=floatfmt
)
self._logger.info("\n{}".format(res_log_tab_str))
errors_cache_path = osp.join(
self._output_dir, f"{method_name}_{self.dataset_name}_errors.pkl")
recalls_cache_path = osp.join(
self._output_dir,
f"{method_name}_{self.dataset_name}_precisions.pkl")
self._logger.info(f"{errors_cache_path}")
self._logger.info(f"{recalls_cache_path}")
mmcv.dump(errors, errors_cache_path)
mmcv.dump(precisions, recalls_cache_path)
dump_tab_name = osp.join(
self._output_dir,
f"{method_name}_{self.dataset_name}_tab_precisions.txt")
with open(dump_tab_name, "w") as f:
f.write("{}\n".format(res_log_tab_str))
if self._distributed:
self._logger.warning(
"\n The current evaluation on multi-gpu is not correct, run with single-gpu instead."
)
return {}
def process_net_and_pnp(self, inputs, outputs, out_dict, pnp_type="iter"):
"""Initialize with network prediction (learned PnP) + iter PnP
Args:
inputs: the inputs to a model.
It is a list of dict. Each dict corresponds to an image and
contains keys like "height", "width", "file_name", "image_id", "scene_id".
pnp_type: iter | ransac (use ransac+EPnP)
outputs:
"""
cfg = self.cfg
out_coor_x = out_dict["coor_x"].detach()
out_coor_y = out_dict["coor_y"].detach()
out_coor_z = out_dict["coor_z"].detach()
out_xyz = get_out_coor(cfg, out_coor_x, out_coor_y, out_coor_z)
out_xyz = out_xyz.to(self._cpu_device).numpy()
out_mask = get_out_mask(cfg, out_dict["mask"].detach())
out_mask = out_mask.to(self._cpu_device).numpy()
out_rots = out_dict["rot"].detach().to(self._cpu_device).numpy()
out_transes = out_dict["trans"].detach().to(self._cpu_device).numpy()
out_i = -1
for i, (_input, output) in enumerate(zip(inputs, outputs)):
start_process_time = time.perf_counter()
for inst_i in range(len(_input["roi_img"])):
out_i += 1
bbox_center_i = _input["bbox_center"][inst_i]
cx_i, cy_i = bbox_center_i
scale_i = _input["scale"][inst_i]
coord_2d_i = _input["roi_coord_2d"][inst_i].cpu().numpy(
).transpose(1, 2, 0) # CHW->HWC
im_H = _input["im_H"][inst_i].item()
im_W = _input["im_W"][inst_i].item()
scene_im_id_split = _input["scene_im_id"][inst_i].split("/")
K = _input["cam"][inst_i].cpu().numpy().copy()
roi_label = _input["roi_cls"][inst_i] # 0-based label
score = _input["score"][inst_i]
roi_label, cls_name = self._maybe_adapt_label_cls_name(
roi_label)
if cls_name is None:
continue
# scene_id = int(scene_im_id_split[0])
scene_id = scene_im_id_split[0]
im_id = int(scene_im_id_split[1])
# get pose
xyz_i = out_xyz[out_i].transpose(1, 2, 0)
mask_i = np.squeeze(out_mask[out_i])
img_points, model_points = self.get_img_model_points_with_coords2d(
mask_i,
xyz_i,
coord_2d_i,
im_H=im_H,
im_W=im_W,
extent=_input["roi_extent"][inst_i].cpu().numpy(),
mask_thr=cfg.MODEL.CDPN.ROT_HEAD.MASK_THR_TEST,
)
rot_est_net = out_rots[out_i]
trans_est_net = out_transes[out_i]
num_points = len(img_points)
if num_points >= 4:
dist_coeffs = np.zeros(shape=[8, 1], dtype="float64")
points_2d = np.ascontiguousarray(
img_points.astype(np.float64))
points_3d = np.ascontiguousarray(
model_points.astype(np.float64))
camera_matrix = K.astype(np.float64)
rvec0, _ = cv2.Rodrigues(rot_est_net)
if pnp_type in ["ransac", "ransac_rot"]:
points_3d = np.expand_dims(points_3d, 0)
points_2d = np.expand_dims(points_2d, 0)
_, rvec, t_est, _ = cv2.solvePnPRansac(
objectPoints=points_3d,
imagePoints=points_2d,
cameraMatrix=camera_matrix,
distCoeffs=dist_coeffs,
flags=cv2.SOLVEPNP_EPNP,
useExtrinsicGuess=True,
rvec=rvec0,
tvec=trans_est_net,
reprojectionError=3.0, # default 8.0
iterationsCount=20,
)
else: # iter PnP
# points_3d = np.expand_dims(points_3d, 0) # uncomment for EPNP
# points_2d = np.expand_dims(points_2d, 0)
_, rvec, t_est = cv2.solvePnP(
objectPoints=points_3d,
imagePoints=points_2d,
cameraMatrix=camera_matrix,
distCoeffs=dist_coeffs,
flags=cv2.SOLVEPNP_ITERATIVE,
# flags=cv2.SOLVEPNP_EPNP,
useExtrinsicGuess=True,
rvec=rvec0,
tvec=trans_est_net,
)
rot_est, _ = cv2.Rodrigues(rvec)
if pnp_type not in ["ransac_rot"]:
diff_t_est = te(t_est, trans_est_net)
if diff_t_est > 1: # diff too large
self._logger.warning(
"translation error too large: {}".format(
diff_t_est))
t_est = trans_est_net
else:
t_est = trans_est_net
pose_est = np.concatenate(
[rot_est, t_est.reshape((3, 1))], axis=-1)
else:
self._logger.warning("num points: {}".format(
len(img_points)))
pose_est_net = np.hstack(
[rot_est_net, trans_est_net.reshape(3, 1)])
pose_est = pose_est_net
output["pose_est"] = pose_est
output["time"] += time.perf_counter() - start_process_time
# result
file_name = _input["file_name"][inst_i]
if cls_name not in self._predictions:
self._predictions[cls_name] = OrderedDict()
result = {
"score": score,
"R": pose_est[:3, :3],
"t": pose_est[:3, 3],
"time": output["time"]
}
self._predictions[cls_name][file_name] = result
proj_2d_err = pose_error.arp_2d_sym(
R_e, t_e, R_g, t_g, pts=models[obj_id]["pts"], K=K, syms=models_sym[obj_id]
)
e = [proj_2d_err]
elif p["error_type"] == "cus":
if sphere_projections_overlap:
e = [
pose_error.cus(R_e, t_e, R_g, t_g, K, ren, obj_id, renderer_type=p["renderer_type"])
]
else:
e = [1.0]
elif p["error_type"] == "rete":
r_err = pose_error.re(R_e, R_g)
t_err = pose_error.te(t_e, t_g) / 10 # mm to cm
e = [r_err, t_err]
elif p["error_type"] == "reteS":
r_err = pose_error.re_sym(R_e, R_g, syms=models_sym[obj_id])
t_err = pose_error.te_sym(t_e, t_g, R_gt=R_g, syms=models_sym[obj_id]) / 10
e = [r_err, t_err]
elif p["error_type"] == "re":
r_err = pose_error.re(R_e, R_g)
e = [r_err]
elif p["error_type"] == "reS":
r_err = pose_error.re_sym(R_e, R_g, syms=models_sym[obj_id])
e = [r_err]