def visualize(self, output, batch):
inp = img_utils.unnormalize_img(batch['inp'][0], mean,
std).permute(1, 2, 0)
kpt_2d = output['kpt_2d'][0].detach().cpu().numpy()
plt.imshow(inp)
plt.plot(kpt_2d[:, 0], kpt_2d[:, 1], '.')
plt.show()
def visualize_ex(self, output, batch):
inp = img_utils.bgr_to_rgb(
img_utils.unnormalize_img(batch['inp'][0], mean,
std).permute(1, 2, 0))
ex = output['py']
ex = ex[-1] if isinstance(ex, list) else ex
ex = ex.detach().cpu().numpy() * snake_config.down_ratio
fig, ax = plt.subplots(1, figsize=(20, 10))
fig.tight_layout()
ax.axis('off')
ax.imshow(inp)
colors = np.array([[31, 119, 180], [255, 127, 14], [46, 160, 44],
[214, 40, 39], [148, 103, 189], [140, 86, 75],
[227, 119, 194], [126, 126, 126], [188, 189, 32],
[26, 190, 207]]) / 255.
np.random.shuffle(colors)
colors = cycle(colors)
for i in range(len(ex)):
color = next(colors).tolist()
poly = ex[i]
poly = np.append(poly, [poly[0]], axis=0)
ax.plot(poly[:, 0], poly[:, 1], color=color)
plt.show()
def visualize_linemod_ann(img, kpt_2d, mask, savefig=False):
img = img_utils.unnormalize_img(img, mean, std, False).permute(1, 2, 0)
_, (ax1, ax2) = plt.subplots(1, 2)
ax1.imshow(img)
ax1.plot(kpt_2d[:, 0], kpt_2d[:, 1], '.')
ax2.imshow(mask)
plt.show()
def visualize_training_box(self, output, batch, img_name):
inp = img_utils.bgr_to_rgb(img_utils.unnormalize_img(batch['inp'][0], mean, std).permute(1, 2, 0))
box = output['detection'][:, :4].detach().cpu().numpy() * snake_config.down_ratio
ex = output['py']
ex = ex[-1] if isinstance(ex, list) else ex
ex = ex.detach().cpu().numpy() * snake_config.down_ratio
#这里是查看显示结果的相关参数
tmp_file = open('/home/tianhao.lu/code/Deep_snake/snake/Result/Contour/result.log', 'w', encoding='utf8')
tmp_file.writelines("visualize training box -> inp：" + str(type(inp)) + "\n")
tmp_file.writelines("visualize training box -> box：" + str(len(box)) + "\n")
tmp_file.writelines("visualize training box -> ex:" + str(ex) + "\n")
tmp_file.writelines("visualize training box -> detection:" + str(output['detection']) + "\n")
# for tmp_data in train_loader:
# tmp_file.writelines("one train_loader data type:" + str(type(tmp_data)) + "\n")
# for key in tmp_data:
# tmp_file.writelines("one train_loader data key:" + str(key) + "\n")
# tmp_file.writelines("one train_loader data len:" + str(len(tmp_data[key])) + "\n")
# # tmp_file.writelines("one train_loader data:" + str(tmp_data) + "\n")
# break
tmp_file.writelines(str("*************************************************************** \n"))
tmp_file.close()
fig, ax = plt.subplots(1, figsize=(20, 10))
fig.tight_layout()
ax.axis('off')
ax.imshow(inp)
colors = np.array([
[31, 119, 180],
[255, 127, 14],
[46, 160, 44],
[214, 40, 39],
[148, 103, 189],
[140, 86, 75],
[227, 119, 194],
[126, 126, 126],
[188, 189, 32],
[26, 190, 207]
]) / 255.
np.random.shuffle(colors)
colors = cycle(colors)
for i in range(len(ex)):
color = next(colors).tolist()
poly = ex[i]
poly = np.append(poly, [poly[0]], axis=0)
ax.plot(poly[:, 0], poly[:, 1], color=color, linewidth=5)
x_min, y_min, x_max, y_max = box[i]
ax.plot([x_min, x_min, x_max, x_max, x_min], [y_min, y_max, y_max, y_min, y_min], color='w', linewidth=0.5)
# filename = random.randint(0,100000) #这个是名称取十万以内的随机数
filename = img_name
# fig = plt.gcf()
img_width = 400
img_height = 300
fig.set_size_inches(img_width / 96.0, img_height / 96.0) # 输出width*height像素
plt.subplots_adjust(top=1, bottom=0, left=0, right=1, hspace=0, wspace=0) # 输出图像#边框设置
plt.margins(0, 0)
plt.savefig("/home/tianhao.lu/code/Deep_snake/snake/Result/coco_test_result/%s.png"%filename,dpi=96.0,pad_inches=0.0)
def visualize(self, output, batch):
inp = img_utils.bgr_to_rgb(img_utils.unnormalize_img(batch['inp'][0], mean, std).permute(1, 2, 0))
box = output['detection'][0, :, :4].detach().cpu().numpy() * tless_config.down_ratio
score = output['detection'][0, :, 4].detach().cpu().numpy()
plt.imshow(inp)
for i in range(len(box)):
x_min, y_min, x_max, y_max = box[i]
plt.plot([x_min, x_min, x_max, x_max, x_min], [y_min, y_max, y_max, y_min, y_min])
plt.show()
def visualize_training_box(self, output, batch):
import cv2
inp = img_utils.bgr_to_rgb(img_utils.unnormalize_img(batch['inp'][0], mean, std).permute(1, 2, 0))
h, w, _ = inp.shape
box = output['detection'][:, :4].detach().cpu().numpy() * snake_config.down_ratio
#box = output['detection'][:, :4].detach().cpu().numpy()
nms_ct_hm = output['nms_ct_hm'].detach().cpu().numpy()
nms_ct_hm = nms_ct_hm[0,0,...]
nms_ct_hm = Image.fromarray(nms_ct_hm)
nms_ct_hm = nms_ct_hm.resize((w, h))
ex = output['py']
ex = ex[-1] if isinstance(ex, list) else ex
ex = ex.detach().cpu().numpy() * snake_config.down_ratio
#ex = ex.detach().cpu().numpy()
fig, ax = plt.subplots(1, figsize=(10, 10))
fig.tight_layout()
ax.axis('off')
ax.imshow(inp)
ax.imshow(np.array(nms_ct_hm))
colors = np.array([
[31, 119, 180],
[255, 127, 14],
[46, 160, 44],
[214, 40, 39],
[148, 103, 189],
[140, 86, 75],
[227, 119, 194],
[126, 126, 126],
[188, 189, 32],
[26, 190, 207]
]) / 255.
np.random.shuffle(colors)
colors = cycle(colors)
for i in range(len(ex)):
color = next(colors).tolist()
poly = ex[i]
poly = np.append(poly, [poly[0]], axis=0)
ax.plot(poly[:, 0], poly[:, 1], color=color, linewidth=1)
x_min, y_min, x_max, y_max = box[i]
ax.plot([x_min, x_min, x_max, x_max, x_min], [y_min, y_max, y_max, y_min, y_min], color='w', linewidth=0.5)
print("box:", box[i])
plt.show()
def visualize_training_box(self, output, batch):
inp = img_utils.bgr_to_rgb(
img_utils.unnormalize_img(batch['inp'][0], mean,
std).permute(1, 2, 0))
box = output['detection'][:, :4].detach().cpu().numpy(
) * snake_config.down_ratio
# box = output['cp_box'][:, :4].detach().cpu().numpy() * snake_config.down_ratio
_, ax = plt.subplots(1)
ax.imshow(inp)
n = len(box)
for i in range(n):
x_min, y_min, x_max, y_max = box[i]
ax.plot([x_min, x_min, x_max, x_max, x_min],
[y_min, y_max, y_max, y_min, y_min])
plt.show()
def visualize_train(self, output, batch):
inp = img_utils.unnormalize_img(batch['inp'][0], mean, std).permute(1, 2, 0)
mask = batch['mask'][0].detach().cpu().numpy()
vertex = batch['vertex'][0][0].detach().cpu().numpy()
img_id = int(batch['img_id'][0])
anno = self.coco.loadAnns(self.coco.getAnnIds(imgIds=img_id))[0]
fps_2d = np.array(anno['fps_2d'])
plt.figure(0)
plt.subplot(221)
plt.imshow(inp)
plt.subplot(222)
plt.imshow(mask)
plt.plot(fps_2d[:, 0], fps_2d[:, 1])
plt.subplot(224)
plt.imshow(vertex)
plt.savefig('test.jpg')
plt.close(0)
def visualize_demo(self, output, inp, meta):
inp = img_utils.unnormalize_img(inp[0], mean, std).permute(1, 2, 0)
kpt_2d = output['kpt_2d'][0].detach().cpu().numpy()
kpt_3d = np.array(meta['kpt_3d'])
K = np.array(meta['K'])
pose_pred = pvnet_pose_utils.pnp(kpt_3d, kpt_2d, K)
corner_3d = np.array(meta['corner_3d'])
corner_2d_pred = pvnet_pose_utils.project(corner_3d, K, pose_pred)
_, ax = plt.subplots(1)
ax.imshow(inp)
ax.add_patch(patches.Polygon(xy=corner_2d_pred[[0, 1, 3, 2, 0, 4, 6, 2]], fill=False, linewidth=1, edgecolor='b'))
ax.add_patch(patches.Polygon(xy=corner_2d_pred[[5, 4, 6, 7, 5, 1, 3, 7]], fill=False, linewidth=1, edgecolor='b'))
plt.show()
def visualize_gt(self, output, batch, index):
inp = img_utils.bgr_to_rgb(
img_utils.unnormalize_img(batch['inp'][0], mean,
std).permute(1, 2, 0))
box = output['detection'][:, :4].detach().cpu().numpy(
) * snake_config.down_ratio
ex = output['i_gt_4py']
ex = ex[-1] if isinstance(ex, list) else ex
ex = ex.detach().cpu().numpy() * snake_config.down_ratio
fig, ax = plt.subplots(1, figsize=(20, 10))
fig.tight_layout()
ax.axis('off')
ax.imshow(inp)
colors = np.array([[31, 119, 180], [255, 127, 14], [46, 160, 44],
[214, 40, 39], [148, 103, 189], [140, 86, 75],
[227, 119, 194], [126, 126, 126], [188, 189, 32],
[26, 190, 207]]) / 255.
np.random.shuffle(colors)
colors = cycle(colors)
for i in range(len(ex)):
color = next(colors).tolist()
poly = ex[i]
poly = np.append(poly, [poly[0]], axis=0)
ax.plot(poly[:, 0], poly[:, 1], color=color, linewidth=3)
x_min, y_min, x_max, y_max = box[i]
ax.plot([x_min, x_min, x_max, x_max, x_min],
[y_min, y_max, y_max, y_min, y_min],
color='w',
linewidth=0.5)
plt.show()
# os.mkdir()
os.makedirs("KINS_SNAKE_GT_VIS/", exist_ok=True)
out_png_path = "KINS_SNAKE_GT_VIS/" + str(index) + ".jpg"
# plt.savefig(out_png_path, format='jpg', transparent=True, dpi=100, pad_inches = 0)
plt.savefig(out_png_path,
format='jpg',
transparent=True,
dpi=100,
bbox_inches='tight')
def visualize(self, output, batch, name, high_resolution=None):
inp = img_utils.unnormalize_img(batch['inp'][0], mean,
std).permute(1, 2, 0)
#inpnew = high_resolution
kpt_2d = output['kpt_2d'][0].detach().cpu().numpy()
#print(kpt_2d)
img_id = int(batch['img_id'][0])
anno = self.coco.loadAnns(self.coco.getAnnIds(imgIds=img_id))[0]
kpt_3d = np.concatenate([anno['fps_3d'], [anno['center_3d']]], axis=0)
#K = np.array(anno['K'])
#print(K)
#print(K.shape)
K = np.array([[1.2430691e+03, 0, 5.45181403e+02],
[0, 1.23942895e+03, 7.0817400e+02], [0, 0, 1]])
#pose_gt = np.array(anno['pose'])
pose_pred = pvnet_pose_utils.pnp(kpt_3d, kpt_2d, K)
corner_3d = np.array(anno['corner_3d'])
#corner_2d_gt = pvnet_pose_utils.project(corner_3d, K, pose_gt)
corner_2d_pred = pvnet_pose_utils.project(corner_3d, K, pose_pred)
#print(corner_2d_pred)
#corner_2d_pred = corner_2d_pred * 2.44
_, ax = plt.subplots(1)
plt.axis('off')
frame = plt.gca()
frame.axes.get_yaxis().set_visible(False)
frame.axes.get_xaxis().set_visible(False)
ax.imshow(inp)
#ax.add_patch(patches.Polygon(xy=corner_2d_gt[[0, 1, 3, 2, 0, 4, 6, 2]], fill=False, linewidth=1, edgecolor='g'))
#ax.add_patch(patches.Polygon(xy=corner_2d_gt[[5, 4, 6, 7, 5, 1, 3, 7]], fill=False, linewidth=1, edgecolor='g'))
ax.add_patch(
patches.Polygon(xy=corner_2d_pred[[0, 1, 3, 2, 0, 4, 6, 2]],
fill=False,
linewidth=2,
edgecolor='r'))
ax.add_patch(
patches.Polygon(xy=corner_2d_pred[[5, 4, 6, 7, 5, 1, 3, 7]],
fill=False,
linewidth=2,
edgecolor='r'))
ax.figure.savefig(name, bbox_inches='tight', dpi=244)
def visualize_ex(self, output, batch):
inp = img_utils.bgr_to_rgb(
img_utils.unnormalize_img(batch['inp'][0], mean,
std).permute(1, 2, 0))
detection = output['detection']
score = detection[:, 4].detach().cpu().numpy()
label = detection[:, 5].detach().cpu().numpy().astype(int)
cp_ind = output['cp_ind'].detach().cpu().numpy().astype(int)
py = output['py'][-1].detach().cpu().numpy() * snake_config.down_ratio
if len(py) == 0:
return
ct_ind = np.unique(cp_ind)
score = score[ct_ind]
label = label[ct_ind]
ind_group = [
np.argwhere(ct_ind[i] == cp_ind).ravel()
for i in range(len(ct_ind))
]
py = [[py[ind] for ind in inds] for inds in ind_group]
fig, ax = plt.subplots(1, figsize=(20, 10))
fig.tight_layout()
ax.axis('off')
ax.imshow(inp)
colors = np.array([[31, 119, 180], [255, 127, 14], [46, 160, 44],
[214, 40, 39], [148, 103, 189], [140, 86, 75],
[227, 119, 194], [126, 126, 126], [188, 189, 32],
[26, 190, 207]]) / 255.
# colors = cycle(colors)
for i in range(len(py)):
color = colors[np.random.randint(len(colors))]
# color = next(colors).tolist()
for poly in py[i]:
poly = np.append(poly, [poly[0]], axis=0)
ax.plot(poly[:, 0], poly[:, 1], color=color, linewidth=3)
plt.show()
def visualize(self, output, batch):
inp = img_utils.unnormalize_img(batch['inp'][0], mean,
std).permute(1, 2, 0)
kpt_2d = output['kpt_2d'][0].detach().cpu().numpy()
img_id = int(batch['img_id'][0])
anno = self.coco.loadAnns(self.coco.getAnnIds(imgIds=img_id))[0]
kpt_3d = np.concatenate([anno['fps_3d'], [anno['center_3d']]], axis=0)
K = np.array(anno['K'])
pose_gt = np.array(anno['pose'])
pose_pred = pvnet_pose_utils.pnp(kpt_3d, kpt_2d, K)
corner_3d = np.array(anno['corner_3d'])
corner_2d_gt = pvnet_pose_utils.project(corner_3d, K, pose_gt)
corner_2d_pred = pvnet_pose_utils.project(corner_3d, K, pose_pred)
_, ax = plt.subplots(1)
ax.imshow(inp)
ax.add_patch(
patches.Polygon(xy=corner_2d_gt[[0, 1, 3, 2, 0, 4, 6, 2]],
fill=False,
linewidth=1,
edgecolor='g'))
ax.add_patch(
patches.Polygon(xy=corner_2d_gt[[5, 4, 6, 7, 5, 1, 3, 7]],
fill=False,
linewidth=1,
edgecolor='g'))
ax.add_patch(
patches.Polygon(xy=corner_2d_pred[[0, 1, 3, 2, 0, 4, 6, 2]],
fill=False,
linewidth=1,
edgecolor='b'))
ax.add_patch(
patches.Polygon(xy=corner_2d_pred[[5, 4, 6, 7, 5, 1, 3, 7]],
fill=False,
linewidth=1,
edgecolor='b'))
plt.show()
def visualize_heatmap_on_img(self, output, batch, vis_file=None):
import cv2
inp = img_utils.bgr_to_rgb(img_utils.unnormalize_img(batch['inp'][0], mean, std).permute(1, 2, 0))
h, w, _ = inp.shape
raw_ct_hm = output['ct_hm'].detach().cpu().numpy()
raw_ct_hm = raw_ct_hm[0,0,...]
raw_ct_hm = Image.fromarray(raw_ct_hm)
raw_ct_hm = raw_ct_hm.resize((w, h))
nms_ct_hm = output['nms_ct_hm'].detach().cpu().numpy()
nms_ct_hm = nms_ct_hm[0,0,...]
nms_ct_hm = Image.fromarray(nms_ct_hm)
nms_ct_hm = nms_ct_hm.resize((w, h))
raw_ct_hm = np.array(raw_ct_hm)
plt.imshow(inp)
plt.imshow(raw_ct_hm, alpha=0.8, cmap='rainbow')
plt.axis('off')
if vis_file is not None:
plt.savefig(vis_file,format='png',dpi=600)
plt.close()
else:
plt.show()
def visualize(self, output, batch, output_file=None):
col_nb = 4
fig = plt.figure(figsize=(12, 3))
axes = fig.subplots(1, 4, gridspec_kw={"left": 0.0, "right": 1.0, "bottom": 0.0, "top": 1.0})
inp = img_utils.unnormalize_img(batch['inp'][0], mean, std).permute(1, 2, 0)
kpt_2d = output['kpt_2d'][0].detach().cpu().numpy()
var_2d = output['var'][0].detach().cpu().numpy()
img_id = int(batch['img_id'][0])
anno = self.coco.loadAnns(self.coco.getAnnIds(imgIds=img_id))[0]
kpt_3d = np.concatenate([anno['fps_3d'], [anno['center_3d']]], axis=0)
K = np.array(anno['K'])
pose_gt = np.array(anno['pose'])
#pose_pred = pvnet_pose_utils.pnp(kpt_3d, kpt_2d, K)
pose_pred = pvnet_pose_utils.uncertainty_pnp(kpt_3d, kpt_2d, var_2d, K)
kpt_2d_gt = pvnet_pose_utils.project(kpt_3d, K, pose_gt)
verts_2d_gt = pvnet_pose_utils.project(np.array(self.mesh.vertices, copy=True), K, pose_gt)
verts_2d_pred = pvnet_pose_utils.project(np.array(self.mesh.vertices, copy=True), K, pose_pred)
rot_verts_3d_gt = np.dot(np.array(self.mesh.vertices, copy=True), pose_gt[:, :3].T) + pose_gt[:, 3:].T
rot_verts_3d_pred = np.dot(np.array(self.mesh.vertices, copy=True), pose_pred[:, :3].T) + pose_pred[:, 3:].T
# Column 0
col_idx = 0
axes[col_idx].imshow(inp)
axes[col_idx].axis("off")
if kpt_2d_gt is not None:
axes[col_idx].scatter(
kpt_2d_gt[:, 0], kpt_2d_gt[:, 1], c="b", s=2, marker="X", alpha=0.7
)
if kpt_2d is not None:
axes[col_idx].scatter(
kpt_2d[:, 0], kpt_2d[:, 1], c="r", s=2, marker="X", alpha=0.7
)
if var_2d is not None:
ells = compute_confidence_ellipses(kpt_2d, var_2d)
for e in ells:
axes[col_idx].add_artist(e)
if kpt_2d_gt is not None and kpt_2d is not None:
arrow_nb = kpt_2d_gt.shape[0]
idxs = range(arrow_nb)
arrows = np.concatenate([kpt_2d_gt[idxs].astype(np.float), kpt_2d[idxs].astype(np.float)], axis=0)
links = [[i, i + arrow_nb] for i in idxs]
_visualize_joints_2d(
axes[col_idx],
arrows,
alpha=0.5,
joint_idxs=False,
links=links,
color=["k"] * arrow_nb,
)
# Column 1
col_idx = 1
axes[col_idx].imshow(inp)
axes[col_idx].axis("off")
# Visualize 2D object vertices
if verts_2d_gt is not None:
axes[col_idx].scatter(
verts_2d_gt[:, 0], verts_2d_gt[:, 1], c="b", s=1, alpha=0.05
)
if verts_2d_pred is not None:
axes[col_idx].scatter(
verts_2d_pred[:, 0], verts_2d_pred[:, 1], c="r", s=1, alpha=0.02
)
# Column 2
# view from the top
col_idx = 2
if rot_verts_3d_gt is not None:
axes[col_idx].scatter(
rot_verts_3d_gt[:, 2], rot_verts_3d_gt[:, 0], c="b", s=1, alpha=0.02
)
if rot_verts_3d_pred is not None:
axes[col_idx].scatter(
rot_verts_3d_pred[:, 2], rot_verts_3d_pred[:, 0], c="r", s=1, alpha=0.02
)
axes[col_idx].invert_yaxis()
# Column 3
# view from the right
col_idx = 3
# invert second axis here for more consistent viewpoints
if rot_verts_3d_gt is not None:
axes[col_idx].scatter(
rot_verts_3d_gt[:, 2], -rot_verts_3d_gt[:, 1], c="b", s=1, alpha=0.02
)
if rot_verts_3d_pred is not None:
axes[col_idx].scatter(
rot_verts_3d_pred[:, 2], -rot_verts_3d_pred[:, 1], c="r", s=1, alpha=0.02
)
_squashfig(fig)
if output_file is None:
plt.show()
else:
fig.savefig(output_file, dpi=300)
plt.close()
