def evaluate(self, output, batch):
kpt_2d = output['kpt_2d'][0].detach().cpu().numpy()
img_id = int(batch['img_id'][0])
anno = self.coco.loadAnns(self.coco.getAnnIds(imgIds=img_id))[0]
kpt_3d = np.concatenate([anno['fps_3d'], [anno['center_3d']]], axis=0)
K = np.array(anno['K'])
pose_gt = np.array(anno['pose'])
if cfg.test.un_pnp:
var = output['var'][0].detach().cpu().numpy()
pose_pred = self.uncertainty_pnp(kpt_3d, kpt_2d, var, K)
else:
pose_pred = pvnet_pose_utils.pnp(kpt_3d, kpt_2d, K)
if cfg.test.icp:
pose_pred_icp = self.icp_refine(pose_pred.copy(), anno, output, K)
if cfg.cls_type in ['eggbox', 'glue']:
self.add_metric(pose_pred_icp, pose_gt, syn=True, icp=True)
else:
self.add_metric(pose_pred_icp, pose_gt, icp=True)
self.projection_2d(pose_pred_icp, pose_gt, K, icp=True)
self.cm_degree_5_metric(pose_pred_icp, pose_gt, icp=True)
if cfg.cls_type in ['eggbox', 'glue']:
self.add_metric(pose_pred, pose_gt, syn=True)
else:
self.add_metric(pose_pred, pose_gt)
self.projection_2d(pose_pred, pose_gt, K)
self.cm_degree_5_metric(pose_pred, pose_gt)
self.mask_iou(output, batch)
def evaluate(self, output, batch):
img_id = int(batch['meta']['img_id'])
self.img_ids.append(img_id)
img_data = self.coco.loadImgs(int(img_id))[0]
depth_path = img_data['depth_path']
ann_ids = self.coco.getAnnIds(imgIds=img_id, catIds=self.obj_id)
annos = self.coco.loadAnns(ann_ids)
kpt_3d = np.concatenate([annos[0]['fps_3d'], [annos[0]['center_3d']]],
axis=0)
corner_3d = np.array(annos[0]['corner_3d'])
K = np.array(annos[0]['K'])
pose_gt = [np.array(anno['pose']) for anno in annos]
kpt_2d = output['kpt_2d'].detach().cpu().numpy()
centers = batch['meta']['center']
scales = batch['meta']['scale']
boxes = batch['meta']['box']
h, w = batch['inp'].size(2), batch['inp'].size(3)
pose_preds = []
pose_preds_icp = []
for i in range(len(centers)):
center = centers[i].detach().cpu().numpy()
scale = scales[i].detach().cpu().numpy()
kpt_2d_ = kpt_2d[i]
trans_inv = data_utils.get_affine_transform(center[0],
scale[0],
0, [w, h],
inv=1)
kpt_2d_ = data_utils.affine_transform(kpt_2d_, trans_inv)
if cfg.test.un_pnp:
var = output['var'][i].detach().cpu().numpy()
pose_pred = self.uncertainty_pnp(kpt_3d, kpt_2d_, var, K)
else:
pose_pred = pvnet_pose_utils.pnp(kpt_3d, kpt_2d_, K)
pose_preds.append(pose_pred)
if cfg.test.icp:
seg = torch.argmax(output['seg'][i],
dim=0).detach().cpu().numpy()
seg = seg.astype(np.uint8)
seg = cv2.warpAffine(seg,
trans_inv, (self.width, self.height),
flags=cv2.INTER_NEAREST)
pose_pred_icp = self.icp_refine(pose_pred.copy(), depth_path,
seg.copy(), K.copy())
pose_preds_icp.append(pose_pred_icp)
if cfg.test.icp:
self.icp_adi.append(self.adi_metric(pose_preds_icp, pose_gt))
self.icp_cmd5.append(
self.cm_degree_5_metric(pose_preds_icp, pose_gt))
self.pose_icp_per_id.append(pose_preds_icp)
self.adi.append(self.adi_metric(pose_preds, pose_gt))
self.cmd5.append(self.cm_degree_5_metric(pose_preds, pose_gt))
self.pose_per_id.append(pose_preds)
def visualize(self, output, batch, id=0):
img = batch['img'][0].detach().cpu().numpy()
center = output['center'][0]
scale = output['scale'][0]
h, w = tless_pvnet_utils.input_scale
trans_output_inv = data_utils.get_affine_transform(center,
scale,
0, [w, h],
inv=1)
kpt = output['kpt_2d'].detach().cpu().numpy()
kpt_2d = data_utils.affine_transform(kpt, trans_output_inv)[0]
img_id = int(batch['img_id'][0])
anno = self.coco.loadAnns(self.coco.getAnnIds(imgIds=img_id))[0]
kpt_3d = np.concatenate([anno['fps_3d'], [anno['center_3d']]], axis=0)
K = np.array(anno['K'])
pose_gt = np.array(anno['pose'])
pose_pred = pvnet_pose_utils.pnp(kpt_3d, kpt_2d, K)
corner_3d = np.array(anno['corner_3d'])
corner_2d_gt = pvnet_pose_utils.project(corner_3d, K, pose_gt)
corner_2d_pred = pvnet_pose_utils.project(corner_3d, K, pose_pred)
_, ax = plt.subplots(1)
ax.imshow(img)
ax.add_patch(
patches.Polygon(xy=corner_2d_gt[[0, 1, 3, 2, 0, 4, 6, 2]],
fill=False,
linewidth=1,
edgecolor='g'))
ax.add_patch(
patches.Polygon(xy=corner_2d_gt[[5, 4, 6, 7, 5, 1, 3, 7]],
fill=False,
linewidth=1,
edgecolor='g'))
ax.add_patch(
patches.Polygon(xy=corner_2d_pred[[0, 1, 3, 2, 0, 4, 6, 2]],
fill=False,
linewidth=1,
edgecolor='b'))
ax.add_patch(
patches.Polygon(xy=corner_2d_pred[[5, 4, 6, 7, 5, 1, 3, 7]],
fill=False,
linewidth=1,
edgecolor='b'))
plt.show()
def visualize_demo(self, output, inp, meta):
inp = img_utils.unnormalize_img(inp[0], mean, std).permute(1, 2, 0)
kpt_2d = output['kpt_2d'][0].detach().cpu().numpy()
kpt_3d = np.array(meta['kpt_3d'])
K = np.array(meta['K'])
pose_pred = pvnet_pose_utils.pnp(kpt_3d, kpt_2d, K)
corner_3d = np.array(meta['corner_3d'])
corner_2d_pred = pvnet_pose_utils.project(corner_3d, K, pose_pred)
_, ax = plt.subplots(1)
ax.imshow(inp)
ax.add_patch(patches.Polygon(xy=corner_2d_pred[[0, 1, 3, 2, 0, 4, 6, 2]], fill=False, linewidth=1, edgecolor='b'))
ax.add_patch(patches.Polygon(xy=corner_2d_pred[[5, 4, 6, 7, 5, 1, 3, 7]], fill=False, linewidth=1, edgecolor='b'))
plt.show()
def visualize(self, output, batch, name, high_resolution=None):
inp = img_utils.unnormalize_img(batch['inp'][0], mean,
std).permute(1, 2, 0)
#inpnew = high_resolution
kpt_2d = output['kpt_2d'][0].detach().cpu().numpy()
#print(kpt_2d)
img_id = int(batch['img_id'][0])
anno = self.coco.loadAnns(self.coco.getAnnIds(imgIds=img_id))[0]
kpt_3d = np.concatenate([anno['fps_3d'], [anno['center_3d']]], axis=0)
#K = np.array(anno['K'])
#print(K)
#print(K.shape)
K = np.array([[1.2430691e+03, 0, 5.45181403e+02],
[0, 1.23942895e+03, 7.0817400e+02], [0, 0, 1]])
#pose_gt = np.array(anno['pose'])
pose_pred = pvnet_pose_utils.pnp(kpt_3d, kpt_2d, K)
corner_3d = np.array(anno['corner_3d'])
#corner_2d_gt = pvnet_pose_utils.project(corner_3d, K, pose_gt)
corner_2d_pred = pvnet_pose_utils.project(corner_3d, K, pose_pred)
#print(corner_2d_pred)
#corner_2d_pred = corner_2d_pred * 2.44
_, ax = plt.subplots(1)
plt.axis('off')
frame = plt.gca()
frame.axes.get_yaxis().set_visible(False)
frame.axes.get_xaxis().set_visible(False)
ax.imshow(inp)
#ax.add_patch(patches.Polygon(xy=corner_2d_gt[[0, 1, 3, 2, 0, 4, 6, 2]], fill=False, linewidth=1, edgecolor='g'))
#ax.add_patch(patches.Polygon(xy=corner_2d_gt[[5, 4, 6, 7, 5, 1, 3, 7]], fill=False, linewidth=1, edgecolor='g'))
ax.add_patch(
patches.Polygon(xy=corner_2d_pred[[0, 1, 3, 2, 0, 4, 6, 2]],
fill=False,
linewidth=2,
edgecolor='r'))
ax.add_patch(
patches.Polygon(xy=corner_2d_pred[[5, 4, 6, 7, 5, 1, 3, 7]],
fill=False,
linewidth=2,
edgecolor='r'))
ax.figure.savefig(name, bbox_inches='tight', dpi=244)
def visualize(self, output, batch):
inp = img_utils.unnormalize_img(batch['inp'][0], mean,
std).permute(1, 2, 0)
kpt_2d = output['kpt_2d'][0].detach().cpu().numpy()
img_id = int(batch['img_id'][0])
anno = self.coco.loadAnns(self.coco.getAnnIds(imgIds=img_id))[0]
kpt_3d = np.concatenate([anno['fps_3d'], [anno['center_3d']]], axis=0)
K = np.array(anno['K'])
pose_gt = np.array(anno['pose'])
pose_pred = pvnet_pose_utils.pnp(kpt_3d, kpt_2d, K)
corner_3d = np.array(anno['corner_3d'])
corner_2d_gt = pvnet_pose_utils.project(corner_3d, K, pose_gt)
corner_2d_pred = pvnet_pose_utils.project(corner_3d, K, pose_pred)
_, ax = plt.subplots(1)
ax.imshow(inp)
ax.add_patch(
patches.Polygon(xy=corner_2d_gt[[0, 1, 3, 2, 0, 4, 6, 2]],
fill=False,
linewidth=1,
edgecolor='g'))
ax.add_patch(
patches.Polygon(xy=corner_2d_gt[[5, 4, 6, 7, 5, 1, 3, 7]],
fill=False,
linewidth=1,
edgecolor='g'))
ax.add_patch(
patches.Polygon(xy=corner_2d_pred[[0, 1, 3, 2, 0, 4, 6, 2]],
fill=False,
linewidth=1,
edgecolor='b'))
ax.add_patch(
patches.Polygon(xy=corner_2d_pred[[5, 4, 6, 7, 5, 1, 3, 7]],
fill=False,
linewidth=1,
edgecolor='b'))
plt.show()
def visualize(self, output, batch):
img_id = int(batch['meta']['img_id'])
img_data = self.coco.loadImgs(int(img_id))[0]
path = img_data['file_name']
depth_path = img_data['depth_path']
img = np.array(Image.open(path))
ann_ids = self.coco.getAnnIds(imgIds=img_id, catIds=self.obj_id)
annos = self.coco.loadAnns(ann_ids)
kpt_3d = np.concatenate([annos[0]['fps_3d'], [annos[0]['center_3d']]],
axis=0)
corner_3d = np.array(annos[0]['corner_3d'])
K = np.array(annos[0]['K'])
kpt_2d = output['kpt_2d'].detach().cpu().numpy()
centers = batch['meta']['center']
scales = batch['meta']['scale']
boxes = batch['meta']['box']
h, w = batch['inp'].size(2), batch['inp'].size(3)
kpt_2ds = []
segs = []
for i in range(len(centers)):
center = centers[i].detach().cpu().numpy()
scale = scales[i].detach().cpu().numpy()
kpt_2d_ = kpt_2d[i]
trans_inv = data_utils.get_affine_transform(center[0],
scale[0],
0, [w, h],
inv=1)
kpt_2d_ = data_utils.affine_transform(kpt_2d_, trans_inv)
kpt_2ds.append(kpt_2d_)
seg = torch.argmax(output['seg'][i], dim=0).detach().cpu().numpy()
seg = seg.astype(np.uint8)
seg = cv2.warpAffine(seg,
trans_inv, (720, 540),
flags=cv2.INTER_NEAREST)
segs.append(seg)
_, ax = plt.subplots(1)
ax.imshow(img)
# for i in range(len(boxes)):
# x_min, y_min, x_max, y_max = boxes[i].view(-1).numpy()
# ax.plot([x_min, x_min, x_max, x_max, x_min], [y_min, y_max, y_max, y_min, y_min])
depth = np.array(Image.open(depth_path)).astype(np.float32)
for i, kpt_2d in enumerate(kpt_2ds):
pose_pred = pvnet_pose_utils.pnp(kpt_3d, kpt_2d, K)
mask = segs[i]
box = cv2.boundingRect(mask.astype(np.uint8))
x, y = box[0] + box[2] / 2., box[1] + box[3] / 2.
z = np.mean(depth[mask != 0] / 10000.)
x = ((x - K[0, 2]) * z) / float(K[0, 0])
y = ((y - K[1, 2]) * z) / float(K[1, 1])
center = [x, y, z]
# pose_pred[:, 3] = center
corner_2d_pred = pvnet_pose_utils.project(corner_3d, K, pose_pred)
ax.add_patch(
patches.Polygon(xy=corner_2d_pred[[0, 1, 3, 2, 0, 4, 6, 2]],
fill=False,
linewidth=1,
edgecolor='b'))
ax.add_patch(
patches.Polygon(xy=corner_2d_pred[[5, 4, 6, 7, 5, 1, 3, 7]],
fill=False,
linewidth=1,
edgecolor='b'))
for anno in annos:
pose_gt = np.array(anno['pose'])
corner_2d_gt = pvnet_pose_utils.project(corner_3d, K, pose_gt)
ax.add_patch(
patches.Polygon(xy=corner_2d_gt[[0, 1, 3, 2, 0, 4, 6, 2]],
fill=False,
linewidth=1,
edgecolor='g'))
ax.add_patch(
patches.Polygon(xy=corner_2d_gt[[5, 4, 6, 7, 5, 1, 3, 7]],
fill=False,
linewidth=1,
edgecolor='g'))
plt.show()