def draw_prediction(self, thedata, args):
import linecache
import re
frame_id = np.random.randint(
1, high=sum(1 for _ in open(self.predict_file, 'r')))
with h5py.File(self.appen_test, 'r') as h5file:
img_id = h5file['index'][frame_id, 0]
# frame_h5 = np.squeeze(h5file['frame'][frame_id, ...], -1)
# poses_h5 = h5file['poses'][frame_id, ...].reshape(-1, 3)
resce_h5 = h5file['resce'][frame_id, ...]
print('[{}] drawing image #{:d} ...'.format(self.name_desc, img_id))
resce3 = resce_h5[0:4]
cube = iso_cube()
cube.load(resce3)
ax = mpplot.subplot(1, 2, 1, projection='3d')
annot_line = args.data_io.get_line(thedata.annotation_train, img_id)
img_name, pose_raw = args.data_io.parse_line_annot(annot_line)
img = args.data_io.read_image(os.path.join(self.image_dir, img_name))
points3 = args.data_ops.img_to_raw(img, self.caminfo)
numpts = points3.shape[0]
if 1000 < numpts:
samid = np.random.choice(numpts, 1000, replace=False)
points3_sam = points3[samid, :]
else:
points3_sam = points3
ax.scatter(points3_sam[:, 0],
points3_sam[:, 1],
points3_sam[:, 2],
color=Color('lightsteelblue').rgb)
ax.view_init(azim=-90, elev=-75)
ax.set_zlabel('depth (mm)', labelpad=15)
args.data_draw.draw_raw3d_pose(ax, thedata, pose_raw)
corners = cube.get_corners()
iso_cube.draw_cube_wire(ax, corners)
mpplot.gca().set_title('Ground truth')
ax = mpplot.subplot(1, 2, 2, projection='3d')
ax.scatter(points3_sam[:, 0],
points3_sam[:, 1],
points3_sam[:, 2],
color=Color('lightsteelblue').rgb)
ax.view_init(azim=-90, elev=-75)
ax.set_zlabel('depth (mm)', labelpad=15)
args.data_draw.draw_raw3d_pose(ax, thedata, pose_raw)
line_pred = linecache.getline(self.predict_file, frame_id)
pred_list = re.split(r'\s+', line_pred.strip())
centre = np.array([float(i) for i in pred_list[1:4]])
cube = iso_cube(centre, self.region_size)
# cube.show_dims()
corners = cube.get_corners()
iso_cube.draw_cube_wire(ax, corners)
mpplot.gca().set_title('Prediction')
print('[{}] drawing image #{:d} - done.'.format(
self.name_desc, img_id))
def generate_anchors_2d(cls, img, pose_raw, anchor_num, caminfo):
""" two sections concatenated:
- positive probability,
- parameters
"""
lattice = latice_image(np.array(img.shape).astype(float), anchor_num)
cube = iso_cube(
(np.max(pose_raw, axis=0) + np.min(pose_raw, axis=0)) / 2,
caminfo.region_size)
cen2d = cls.raw_to_2d(cube.cen.reshape(1, -1), caminfo)
rect = cube.proj_to_rect(caminfo.region_size, cls.raw_to_2d, caminfo)
pcnt = lattice.fill(cen2d) # only one-shot here
anchors = lattice.prow_anchor_single(cen2d, rect.sidelen / 2)
# import matplotlib.pyplot as mpplot
# print(cen2d, rect.sidelen / 2)
# index = np.array(np.unravel_index(np.argmax(pcnt), pcnt.shape))
# print(lattice.yank_anchor_single(
# index,
# anchors
# ))
# mpplot.imshow(img, cmap=mpplot.cm.bone_r)
# rect.show_dims()
# rect.draw(ax)
# mpplot.show()
resce = cube.dump()
return np.append(pcnt.flatten(), anchors), resce
def prow_pose_hit(args, thedata, mode, batch_data):
bi, poses, resce, = \
args[0], args[1], args[2]
cube = iso_cube()
cube.load(resce)
pose_hit = dataops.raw_to_vxhit(poses.reshape(-1, 3), cube, thedata)
batch_data[bi, ...] = pose_hit
def prow_vxudir(args, thedata, mode, batch_data):
bi, pcnt3, poses, resce, = \
args[0], args[1], args[2], args[3]
cube = iso_cube()
cube.load(resce)
vxudir = dataops.raw_to_vxudir(pcnt3, poses.reshape(-1, 3), cube, thedata)
batch_data[bi, ...] = vxudir
def prow_hmap2(args, thedata, mode, batch_data):
bi, poses, resce, = \
args[0], args[1], args[2]
cube = iso_cube()
cube.load(resce)
hmap2 = dataops.raw_to_heatmap2(poses.reshape(-1, 3), cube, thedata)
batch_data[bi, ...] = hmap2
def prow_udir2(args, thedata, mode, batch_data):
bi, clean, poses, resce, = \
args[0], args[1], args[2], args[3]
cube = iso_cube()
cube.load(resce)
udir2 = dataops.raw_to_udir2(clean, poses.reshape(-1, 3), cube, thedata)
batch_data[bi, ...] = udir2
def prow_edt2(args, thedata, mode, batch_data):
bi, clean, poses, resce = \
args[0], args[1], args[2], args[3]
cube = iso_cube()
cube.load(resce)
edt2 = dataops.prop_edt2(clean, poses.reshape(-1, 3), cube, thedata)
batch_data[bi, ...] = edt2
def prow_ov3edt2(args, thedata, mode, batch_data):
bi, ortho3, poses, resce = \
args[0], args[1], args[2], args[3]
cube = iso_cube()
cube.load(resce)
ov3edt2 = dataops.prop_ov3edt2(ortho3, poses.reshape(-1, 3), cube, thedata)
batch_data[bi, ...] = ov3edt2
def fill_grid(cls, img, pose_raw, caminfo):
cube = iso_cube(
(np.max(pose_raw, axis=0) + np.min(pose_raw, axis=0)) / 2,
caminfo.region_size)
pcnt3 = cls.to_pcnt3(img, cube, caminfo)
resce = cube.dump()
return pcnt3, resce
def voxel_hit(cls, img, pose_raw, step, caminfo):
cube = iso_cube(
(np.max(pose_raw, axis=0) + np.min(pose_raw, axis=0)) / 2,
caminfo.region_size)
pcnt3 = cls.to_vxhit(img, cube, caminfo)
resce = cube.dump()
return pcnt3, resce
def prow_pose_c(args, thedata, mode, batch_data):
bi, poses, resce, = \
args[0], args[1], args[2]
cube = iso_cube()
cube.load(resce)
pose_c = cube.transform_to_center(poses.reshape(-1, 3))
batch_data[bi, ...] = pose_c.flatten()
def proj_ortho3(cls, img, pose_raw, caminfo, sort=False):
cube = iso_cube(
(np.max(pose_raw, axis=0) + np.min(pose_raw, axis=0)) / 2,
caminfo.region_size)
img_ortho3 = cls.to_ortho3(img, cube, caminfo, sort=sort)
resce = cube.dump()
return img_ortho3, resce
def prow_vxedt(args, thedata, batch_data):
bi, vxhit, poses, resce = \
args[0], args[1], args[2], args[3]
cube = iso_cube()
cube.load(resce)
vxedt = dataops.prop_edt3(vxhit, poses.reshape(-1, 3), cube, thedata)
batch_data[bi, ...] = vxedt
def prow_pose_c1(args, thedata, batch_data):
bi, poses, resce, = \
args[0], args[1], args[2]
cube = iso_cube()
cube.load(resce)
pose_c1 = cube.transform_center_shrink(poses.reshape(-1, 3))
batch_data[bi, ...] = pose_c1.flatten()
def yanker_hmap(self, resce, hmap2, olmap, uomap, depth, caminfo):
cube = iso_cube()
cube.load(resce)
# return self.data_module.ops.offset_to_raw( # often sum weight to 0
# hmap2, olmap, uomap, depth, cube, caminfo)
return self.data_module.ops.udir2_to_raw(olmap, uomap, depth, cube,
caminfo)
def _debug_draw_prediction(self, did, pred_val):
frame_h5 = np.squeeze(self.batch_data['batch_frame'][did, ...], -1)
resce_h5 = self.batch_data['batch_resce'][did, ...]
import matplotlib.pyplot as mpplot
fig = mpplot.figure(figsize=(2 * 5, 1 * 5))
frame_h5 = self.args.data_ops.frame_size_localizer(
frame_h5, self.caminfo)
colors = [Color('orange').rgb, Color('red').rgb, Color('lime').rgb]
ax = mpplot.subplot(1, 2, 1)
ax.imshow(frame_h5, cmap=mpplot.cm.bone_r)
resce3 = resce_h5[0:4]
cube = iso_cube()
cube.load(resce3)
cube.show_dims()
rects = cube.proj_rects_3(self.args.data_ops.raw_to_2d, self.caminfo)
for ii, rect in enumerate(rects):
rect.draw(ax, colors[ii])
mpplot.gca().set_title('Ground truth')
ax = mpplot.subplot(1, 2, 2)
ax.imshow(frame_h5, cmap=mpplot.cm.bone_r)
cube, index, confidence = self.convert_output(pred_val, self.args,
self.caminfo)
cube.show_dims()
rects = cube.proj_rects_3(self.args.data_ops.raw_to_2d, self.caminfo)
for ii, rect in enumerate(rects):
rect.draw(ax, colors[ii])
mpplot.gca().set_title('Prediction')
fig.tight_layout()
fname = 'debug_train_{}.png'.format(self.name_desc)
mpplot.savefig(os.path.join(self.predict_dir, fname))
mpplot.close(fig)
def draw_random(self, thedata, args):
# mode = 'train'
mode = 'test'
store_handle = self.store_handle[mode]
index_h5 = store_handle['index']
store_size = index_h5.shape[0]
frame_id = np.random.choice(store_size)
# frame_id = 0 # frame_id = img_id - 1
img_id = index_h5[frame_id, ...]
frame_h5 = store_handle['ortho3'][frame_id, ...]
poses_h5 = store_handle['pose_c'][frame_id, ...].reshape(-1, 3)
resce_h5 = store_handle['resce'][frame_id, ...]
print('[{}] drawing image #{:d} ...'.format(self.name_desc, img_id))
from colour import Color
colors = [Color('orange').rgb, Color('red').rgb, Color('lime').rgb]
fig, _ = mpplot.subplots(nrows=2, ncols=3, figsize=(3 * 5, 2 * 5))
resce3 = resce_h5[0:4]
cube = iso_cube()
cube.load(resce3)
# need to maintain both image and poses at the same scale
for spi in range(3):
ax = mpplot.subplot(2, 3, spi + 4)
img = frame_h5[..., spi]
ax.imshow(img, cmap=mpplot.cm.bone_r)
# pose3d = poses_h5
pose3d = cube.trans_scale_to(poses_h5)
pose2d, _ = cube.project_ortho(pose3d, roll=spi, sort=False)
pose2d *= self.crop_size
args.data_draw.draw_pose2d(
ax, thedata,
pose2d,
)
# ax.axis('off')
ax = mpplot.subplot(2, 3, 1)
img_name = args.data_io.index2imagename(img_id)
img = args.data_io.read_image(self.data_inst.images_join(img_name, mode))
ax.imshow(img, cmap=mpplot.cm.bone_r)
pose_raw = self.yanker(poses_h5, resce_h5, self.caminfo)
args.data_draw.draw_pose2d(
ax, thedata,
args.data_ops.raw_to_2d(pose_raw, thedata)
)
rects = cube.proj_rects_3(
args.data_ops.raw_to_2d, self.caminfo
)
for ii, rect in enumerate(rects):
rect.draw(ax, colors[ii])
fig.tight_layout()
mpplot.savefig(os.path.join(
self.predict_dir,
'draw_{}_{}.png'.format(self.name_desc, img_id)))
if self.args.show_draw:
mpplot.show()
print('[{}] drawing image #{:d} - done.'.format(
self.name_desc, img_id))
def draw_raw3d(cls, thedata, img, pose_raw):
cube = iso_cube()
cube.build(pose_raw)
# draw full image
fig_size = (2 * 6, 6)
fig = mpplot.figure(figsize=fig_size)
points3 = dataops.img_to_raw(img, thedata)
numpts = points3.shape[0]
if 1000 < numpts:
samid = np.random.choice(numpts, 1000, replace=False)
points3_sam = points3[samid, :]
else:
points3_sam = points3
ax = fig.add_subplot(1, 2, 1, projection='3d')
ax.scatter(points3_sam[:, 0],
points3_sam[:, 1],
points3_sam[:, 2],
color=Color('lightsteelblue').rgb)
ax.view_init(azim=-90, elev=-75)
ax.set_zlabel('depth (mm)', labelpad=15)
cls.draw_raw3d_pose(ax, thedata, pose_raw)
corners = cube.get_corners()
corners = cube.transform_add_center(corners)
cube.draw_wire(corners)
# draw cropped region
ax = fig.add_subplot(1, 2, 2, projection='3d')
points3_trans = cube.pick(points3)
points3_trans = cube.transform_to_center(points3_trans)
numpts = points3_trans.shape[0]
if 1000 < numpts:
points3_sam = points3_trans[
np.random.choice(numpts, 1000, replace=False), :]
else:
points3_sam = points3_trans
points3_sam = cube.transform_to_center(points3_sam)
pose_trans = cube.transform_to_center(pose_raw)
ax.scatter(points3_sam[:, 0],
points3_sam[:, 1],
points3_sam[:, 2],
color=Color('lightsteelblue').rgb)
cls.draw_raw3d_pose(ax, thedata, pose_trans)
corners = cube.get_corners()
cube.draw_wire(corners)
ax.view_init(azim=-120, elev=-150)
mpplot.tight_layout()
mpplot.show()
# draw projected image
fig_size = (3 * 5, 5)
mpplot.subplots(nrows=1, ncols=3, figsize=fig_size)
for spi in range(3):
ax = mpplot.subplot(1, 3, spi + 1)
coord, depth = cube.project_ortho(points3_trans, roll=spi)
img = cube.print_image(coord, depth, thedata.crop_size)
pose2d, _ = cube.project_ortho(pose_trans, roll=spi, sort=False)
cls.draw_pose2d(ax, thedata, pose2d)
ax.imshow(img, cmap=mpplot.cm.bone_r)
ax.axis('off')
mpplot.tight_layout()
mpplot.show()
def crop_resize(cls, img, pose_raw, caminfo):
# cube.build(pose_raw)
cube = iso_cube(
(np.max(pose_raw, axis=0) + np.min(pose_raw, axis=0)) / 2,
caminfo.region_size)
img_crop = cls.to_crop2(img, cube, caminfo)
resce = cube.dump()
return img_crop, resce
def crop_resize_pca(cls, img, pose_raw, caminfo, sort=False):
# return np.zeros((caminfo.crop_size, caminfo.crop_size)), np.ones(4)
cube = iso_cube(
(np.max(pose_raw, axis=0) + np.min(pose_raw, axis=0)) / 2,
caminfo.region_size)
img_clean = cls.to_clean(img, cube, caminfo, sort=sort)
resce = cube.dump()
return img_clean, resce
def simp_crop(self, dimg):
caminfo = self.caminfo
dimg_f = cv2.bilateralFilter(dimg.astype(np.float32), 5, 30, 30)
dlist = dimg_f.ravel()
if 10 > len(dlist):
self.estr = "hand out of detection range"
print(self.estr)
self.tracker.clear()
return False
dpart10 = np.partition(dlist[caminfo.z_range[0] + 0.01 < dlist], 10)
z0 = dpart10[9]
# print(dlist.shape, np.min(dlist), np.max(dlist))
# print("10th closest point: {}".format(z0)) # 193.10437004605774
if z0 > caminfo.crop_range:
self.estr = "hand out of detection range"
print(self.estr)
self.tracker.clear()
return False
zrs = z0 + caminfo.region_size
in_id = np.where(np.logical_and(z0 - 0.01 < dlist, dlist < zrs))
if 10 > len(in_id[0]):
self.estr = "not enough points in range"
print(self.estr)
self.tracker.clear()
return False
xin, yin = np.unravel_index(in_id, dimg_f.shape)
## FetchHands17!! {
# p2z = np.vstack((yin, xin, dlist[in_id])).T
## }
## live stream {
p2z = np.vstack((xin, yin, dlist[in_id])).T
## }
p3d = self.args.data_ops.d2z_to_raw(p2z, self.caminfo)
## FetchHands17!! {
# cube = iso_cube()
# cube.extent_center(p3d) # FetchHands17!!
# cube.sidelen = self.caminfo.region_size
## }
## find center {
# cen = self.cen_ext.simple_mean(p3d)
cen = self.cen_ext.shape_prior(p3d)
if cen is False:
self.estr = "center not found"
print(self.estr)
self.tracker.clear()
return False
cen_m = self.tracker.update(cen)
# cen_m = cen
print(cen, cen_m)
if cen_m is False:
self.estr = "lost track"
print(self.estr)
self.tracker.clear()
return False
cube = iso_cube(cen_m, self.caminfo.region_size)
## }
# # print(cube.dump()) # [120.0000 -158.0551 -116.6658 240.0000]
return cube
def convert_output(self, pred_val, args, caminfo):
pred_val = pred_val.flatten()
halflen = self.crop_range
centre = np.append(
pred_val[:2] * halflen,
pred_val[2] * halflen + halflen,
)
cube = iso_cube(centre, self.region_size)
return cube
def prow_clean(args, thedata, mode, batch_data):
bi, index, resce = \
args[0], args[1], args[2]
img_name = dataio.index2imagename(index)
img = dataio.read_image(thedata.images_join(img_name, mode))
cube = iso_cube()
cube.load(resce)
img_clean = dataops.to_clean(img, cube, thedata)
batch_data[bi, ...] = img_clean
def test_camera(cap):
# test the camera projection: center should align with the image dimension
cube = iso_cube(np.array([0, 0, 400]), 120)
rects = cube.proj_rects_3(
cap.args.data_ops.raw_to_2d,
cap.caminfo
)
np.set_printoptions(formatter={'float': '{:6.4f}'.format})
for ii, rect in enumerate(rects):
rect.show_dims()
def _draw_image_pose(self, ax, frame, poses, resce, caminfo):
cube = iso_cube()
cube.load(resce)
ax.imshow(frame, cmap=mpplot.cm.bone_r)
pose2d, _ = cube.raw_to_unit(poses)
pose2d *= caminfo.crop_size
self.args.data_draw.draw_pose2d(
ax,
caminfo,
pose2d,
)
def draw_prediction(self, thedata, args):
import linecache
import re
frame_id = np.random.randint(
1, high=sum(1 for _ in open(self.predict_file, 'r')))
with h5py.File(self.appen_test, 'r') as h5file:
img_id = h5file['index'][frame_id, 0]
frame_h5 = np.squeeze(h5file['frame'][frame_id, ...], -1)
# poses_h5 = h5file['poses'][frame_id, ...]
resce_h5 = h5file['resce'][frame_id, ...]
frame_h5 = args.data_ops.frame_size_localizer(
frame_h5, self.caminfo)
print('[{}] drawing image #{:d} ...'.format(self.name_desc, img_id))
colors = [Color('orange').rgb, Color('red').rgb, Color('lime').rgb]
ax = mpplot.subplot(1, 2, 1)
annot_line = args.data_io.get_line(thedata.annotation_train, img_id)
img_name, _ = args.data_io.parse_line_annot(annot_line)
img = args.data_io.read_image(os.path.join(self.image_dir, img_name))
ax.imshow(img, cmap=mpplot.cm.bone_r)
resce3 = resce_h5[0:4]
cube = iso_cube()
cube.load(resce3)
cube.show_dims()
rects = cube.proj_rects_3(args.data_ops.raw_to_2d, self.caminfo)
for ii, rect in enumerate(rects):
rect.draw(ax, colors[ii])
mpplot.gca().set_title('Ground truth')
ax = mpplot.subplot(1, 2, 2)
img = frame_h5
ax.imshow(img, cmap=mpplot.cm.bone_r)
line_pred = linecache.getline(self.predict_file, frame_id)
pred_list = re.split(r'\s+', line_pred.strip())
centre = np.array([float(i) for i in pred_list[1:4]])
cube = iso_cube(centre, self.region_size)
cube.show_dims()
rects = cube.proj_rects_3(args.data_ops.raw_to_2d, self.caminfo)
for ii, rect in enumerate(rects):
rect.draw(ax, colors[ii])
mpplot.gca().set_title('Prediction')
def update(i):
print("==== Frame: ", i, "====")
canvas = self.canvas
camframes = self.cam.provide()
if camframes is None:
return
depth_image = camframes.depth
color_image = camframes.color
canvas.ims[0].set_data(depth_image)
canvas.ims[1].set_data(color_image)
cube = iso_cube(np.array([0, 0, 400]), 120)
# cube=iso_cube(np.array([-200, 20, 400]), 120)
self.show_results(canvas, cube)
def prow_index(args, thedata, batch_data):
bi, line = \
args[0], args[1]
img_name, pose_raw = dataio.parse_line_annot(line)
pose2d = dataops.raw_to_2d(pose_raw, thedata)
if (0 > np.min(pose2d)) or (0 > np.min(thedata.image_size - pose2d)):
return
index = dataio.imagename2index(img_name)
cube = iso_cube((np.max(pose_raw, axis=0) + np.min(pose_raw, axis=0)) / 2,
thedata.region_size)
batch_data['valid'][bi] = True
batch_data['index'][bi, ...] = index
batch_data['poses'][bi, ...] = pose_raw
batch_data['resce'][bi, ...] = cube.dump()
def prow_index(args, thedata, mode, batch_data):
bi, index, poses = \
args[0], args[1], args[2]
pose_raw = poses.reshape(-1, 3)
# pose_raw[:, [0, 1]] = pose_raw[:, [1, 0]]
pose2d = dataops.raw_to_2d(pose_raw, thedata)
if (0 > np.min(pose2d)) or (0 > np.min(thedata.image_size - pose2d)):
return
cube = iso_cube((np.max(pose_raw, axis=0) + np.min(pose_raw, axis=0)) / 2,
thedata.region_size)
batch_data['valid'][bi] = True
batch_data['index'][bi, ...] = index
batch_data['poses'][bi, ...] = pose_raw
batch_data['resce'][bi, ...] = cube.dump()
def voxelize_depth(cls, img, pose_raw, step, anchor_num, caminfo):
halflen = caminfo.crop_range
points3 = cls.img_to_raw(img, caminfo, halflen)
grid = regu_grid(np.array([-halflen, -halflen, caminfo.z_range[0]]),
step, halflen * 2 / step)
pcnt = grid.fill(points3)
cube = iso_cube(
(np.max(pose_raw, axis=0) + np.min(pose_raw, axis=0)) / 2,
caminfo.region_size)
grid.step = anchor_num
grid.cellen = halflen * 2 / anchor_num
anchors = grid.prow_anchor_single(cube.cen, caminfo.region_size)
cubecen = grid.fill([cube.cen])
cubecen_anchors = np.append(cubecen.flatten(), anchors)
resce = cube.dump()
# mpplot = import_module('matplotlib.pyplot')
# print(np.histogram(pcnt))
# grid.show_dims()
# cube.show_dims()
# index = np.array(np.unravel_index(np.argmax(cubecen), cubecen.shape))
# print(index)
# print(grid.yank_anchor_single(
# index,
# anchors
# ))
# ax = mpplot.subplot(projection='3d')
# numpts = points3.shape[0]
# if 1000 < numpts:
# samid = np.random.choice(numpts, 1000, replace=False)
# points3_sam = points3[samid, :]
# else:
# points3_sam = points3
# ax.scatter(
# points3_sam[:, 0], points3_sam[:, 1], points3_sam[:, 2])
# ax.view_init(azim=-90, elev=-75)
# ax.set_zlabel('depth (mm)', labelpad=15)
# corners = cube.get_corners()
# iso_cube.draw_cube_wire(ax, corners)
# from mayavi import mlab
# mlab.figure(size=(800, 800))
# mlab.pipeline.volume(mlab.pipeline.scalar_field(pcnt))
# mlab.pipeline.image_plane_widget(
# mlab.pipeline.scalar_field(pcnt),
# plane_orientation='z_axes',
# slice_index=halflen)
# np.set_printoptions(precision=4)
# mlab.outline()
# mpplot.show()
return pcnt, cubecen_anchors, resce
