def draw_prediction(self, thedata, args): import linecache import re frame_id = np.random.randint( 1, high=sum(1 for _ in open(self.predict_file, 'r'))) with h5py.File(self.appen_test, 'r') as h5file: img_id = h5file['index'][frame_id, 0] # frame_h5 = np.squeeze(h5file['frame'][frame_id, ...], -1) # poses_h5 = h5file['poses'][frame_id, ...].reshape(-1, 3) resce_h5 = h5file['resce'][frame_id, ...] print('[{}] drawing image #{:d} ...'.format(self.name_desc, img_id)) resce3 = resce_h5[0:4] cube = iso_cube() cube.load(resce3) ax = mpplot.subplot(1, 2, 1, projection='3d') annot_line = args.data_io.get_line(thedata.annotation_train, img_id) img_name, pose_raw = args.data_io.parse_line_annot(annot_line) img = args.data_io.read_image(os.path.join(self.image_dir, img_name)) points3 = args.data_ops.img_to_raw(img, self.caminfo) numpts = points3.shape[0] if 1000 < numpts: samid = np.random.choice(numpts, 1000, replace=False) points3_sam = points3[samid, :] else: points3_sam = points3 ax.scatter(points3_sam[:, 0], points3_sam[:, 1], points3_sam[:, 2], color=Color('lightsteelblue').rgb) ax.view_init(azim=-90, elev=-75) ax.set_zlabel('depth (mm)', labelpad=15) args.data_draw.draw_raw3d_pose(ax, thedata, pose_raw) corners = cube.get_corners() iso_cube.draw_cube_wire(ax, corners) mpplot.gca().set_title('Ground truth') ax = mpplot.subplot(1, 2, 2, projection='3d') ax.scatter(points3_sam[:, 0], points3_sam[:, 1], points3_sam[:, 2], color=Color('lightsteelblue').rgb) ax.view_init(azim=-90, elev=-75) ax.set_zlabel('depth (mm)', labelpad=15) args.data_draw.draw_raw3d_pose(ax, thedata, pose_raw) line_pred = linecache.getline(self.predict_file, frame_id) pred_list = re.split(r'\s+', line_pred.strip()) centre = np.array([float(i) for i in pred_list[1:4]]) cube = iso_cube(centre, self.region_size) # cube.show_dims() corners = cube.get_corners() iso_cube.draw_cube_wire(ax, corners) mpplot.gca().set_title('Prediction') print('[{}] drawing image #{:d} - done.'.format( self.name_desc, img_id))
def generate_anchors_2d(cls, img, pose_raw, anchor_num, caminfo): """ two sections concatenated: - positive probability, - parameters """ lattice = latice_image(np.array(img.shape).astype(float), anchor_num) cube = iso_cube( (np.max(pose_raw, axis=0) + np.min(pose_raw, axis=0)) / 2, caminfo.region_size) cen2d = cls.raw_to_2d(cube.cen.reshape(1, -1), caminfo) rect = cube.proj_to_rect(caminfo.region_size, cls.raw_to_2d, caminfo) pcnt = lattice.fill(cen2d) # only one-shot here anchors = lattice.prow_anchor_single(cen2d, rect.sidelen / 2) # import matplotlib.pyplot as mpplot # print(cen2d, rect.sidelen / 2) # index = np.array(np.unravel_index(np.argmax(pcnt), pcnt.shape)) # print(lattice.yank_anchor_single( # index, # anchors # )) # mpplot.imshow(img, cmap=mpplot.cm.bone_r) # rect.show_dims() # rect.draw(ax) # mpplot.show() resce = cube.dump() return np.append(pcnt.flatten(), anchors), resce
def prow_pose_hit(args, thedata, mode, batch_data): bi, poses, resce, = \ args[0], args[1], args[2] cube = iso_cube() cube.load(resce) pose_hit = dataops.raw_to_vxhit(poses.reshape(-1, 3), cube, thedata) batch_data[bi, ...] = pose_hit
def prow_vxudir(args, thedata, mode, batch_data): bi, pcnt3, poses, resce, = \ args[0], args[1], args[2], args[3] cube = iso_cube() cube.load(resce) vxudir = dataops.raw_to_vxudir(pcnt3, poses.reshape(-1, 3), cube, thedata) batch_data[bi, ...] = vxudir
def prow_hmap2(args, thedata, mode, batch_data): bi, poses, resce, = \ args[0], args[1], args[2] cube = iso_cube() cube.load(resce) hmap2 = dataops.raw_to_heatmap2(poses.reshape(-1, 3), cube, thedata) batch_data[bi, ...] = hmap2
def prow_udir2(args, thedata, mode, batch_data): bi, clean, poses, resce, = \ args[0], args[1], args[2], args[3] cube = iso_cube() cube.load(resce) udir2 = dataops.raw_to_udir2(clean, poses.reshape(-1, 3), cube, thedata) batch_data[bi, ...] = udir2
def prow_edt2(args, thedata, mode, batch_data): bi, clean, poses, resce = \ args[0], args[1], args[2], args[3] cube = iso_cube() cube.load(resce) edt2 = dataops.prop_edt2(clean, poses.reshape(-1, 3), cube, thedata) batch_data[bi, ...] = edt2
def prow_ov3edt2(args, thedata, mode, batch_data): bi, ortho3, poses, resce = \ args[0], args[1], args[2], args[3] cube = iso_cube() cube.load(resce) ov3edt2 = dataops.prop_ov3edt2(ortho3, poses.reshape(-1, 3), cube, thedata) batch_data[bi, ...] = ov3edt2
def fill_grid(cls, img, pose_raw, caminfo): cube = iso_cube( (np.max(pose_raw, axis=0) + np.min(pose_raw, axis=0)) / 2, caminfo.region_size) pcnt3 = cls.to_pcnt3(img, cube, caminfo) resce = cube.dump() return pcnt3, resce
def voxel_hit(cls, img, pose_raw, step, caminfo): cube = iso_cube( (np.max(pose_raw, axis=0) + np.min(pose_raw, axis=0)) / 2, caminfo.region_size) pcnt3 = cls.to_vxhit(img, cube, caminfo) resce = cube.dump() return pcnt3, resce
def prow_pose_c(args, thedata, mode, batch_data): bi, poses, resce, = \ args[0], args[1], args[2] cube = iso_cube() cube.load(resce) pose_c = cube.transform_to_center(poses.reshape(-1, 3)) batch_data[bi, ...] = pose_c.flatten()
def proj_ortho3(cls, img, pose_raw, caminfo, sort=False): cube = iso_cube( (np.max(pose_raw, axis=0) + np.min(pose_raw, axis=0)) / 2, caminfo.region_size) img_ortho3 = cls.to_ortho3(img, cube, caminfo, sort=sort) resce = cube.dump() return img_ortho3, resce
def prow_vxedt(args, thedata, batch_data): bi, vxhit, poses, resce = \ args[0], args[1], args[2], args[3] cube = iso_cube() cube.load(resce) vxedt = dataops.prop_edt3(vxhit, poses.reshape(-1, 3), cube, thedata) batch_data[bi, ...] = vxedt
def prow_pose_c1(args, thedata, batch_data): bi, poses, resce, = \ args[0], args[1], args[2] cube = iso_cube() cube.load(resce) pose_c1 = cube.transform_center_shrink(poses.reshape(-1, 3)) batch_data[bi, ...] = pose_c1.flatten()
def yanker_hmap(self, resce, hmap2, olmap, uomap, depth, caminfo): cube = iso_cube() cube.load(resce) # return self.data_module.ops.offset_to_raw( # often sum weight to 0 # hmap2, olmap, uomap, depth, cube, caminfo) return self.data_module.ops.udir2_to_raw(olmap, uomap, depth, cube, caminfo)
def _debug_draw_prediction(self, did, pred_val): frame_h5 = np.squeeze(self.batch_data['batch_frame'][did, ...], -1) resce_h5 = self.batch_data['batch_resce'][did, ...] import matplotlib.pyplot as mpplot fig = mpplot.figure(figsize=(2 * 5, 1 * 5)) frame_h5 = self.args.data_ops.frame_size_localizer( frame_h5, self.caminfo) colors = [Color('orange').rgb, Color('red').rgb, Color('lime').rgb] ax = mpplot.subplot(1, 2, 1) ax.imshow(frame_h5, cmap=mpplot.cm.bone_r) resce3 = resce_h5[0:4] cube = iso_cube() cube.load(resce3) cube.show_dims() rects = cube.proj_rects_3(self.args.data_ops.raw_to_2d, self.caminfo) for ii, rect in enumerate(rects): rect.draw(ax, colors[ii]) mpplot.gca().set_title('Ground truth') ax = mpplot.subplot(1, 2, 2) ax.imshow(frame_h5, cmap=mpplot.cm.bone_r) cube, index, confidence = self.convert_output(pred_val, self.args, self.caminfo) cube.show_dims() rects = cube.proj_rects_3(self.args.data_ops.raw_to_2d, self.caminfo) for ii, rect in enumerate(rects): rect.draw(ax, colors[ii]) mpplot.gca().set_title('Prediction') fig.tight_layout() fname = 'debug_train_{}.png'.format(self.name_desc) mpplot.savefig(os.path.join(self.predict_dir, fname)) mpplot.close(fig)
def draw_random(self, thedata, args): # mode = 'train' mode = 'test' store_handle = self.store_handle[mode] index_h5 = store_handle['index'] store_size = index_h5.shape[0] frame_id = np.random.choice(store_size) # frame_id = 0 # frame_id = img_id - 1 img_id = index_h5[frame_id, ...] frame_h5 = store_handle['ortho3'][frame_id, ...] poses_h5 = store_handle['pose_c'][frame_id, ...].reshape(-1, 3) resce_h5 = store_handle['resce'][frame_id, ...] print('[{}] drawing image #{:d} ...'.format(self.name_desc, img_id)) from colour import Color colors = [Color('orange').rgb, Color('red').rgb, Color('lime').rgb] fig, _ = mpplot.subplots(nrows=2, ncols=3, figsize=(3 * 5, 2 * 5)) resce3 = resce_h5[0:4] cube = iso_cube() cube.load(resce3) # need to maintain both image and poses at the same scale for spi in range(3): ax = mpplot.subplot(2, 3, spi + 4) img = frame_h5[..., spi] ax.imshow(img, cmap=mpplot.cm.bone_r) # pose3d = poses_h5 pose3d = cube.trans_scale_to(poses_h5) pose2d, _ = cube.project_ortho(pose3d, roll=spi, sort=False) pose2d *= self.crop_size args.data_draw.draw_pose2d( ax, thedata, pose2d, ) # ax.axis('off') ax = mpplot.subplot(2, 3, 1) img_name = args.data_io.index2imagename(img_id) img = args.data_io.read_image(self.data_inst.images_join(img_name, mode)) ax.imshow(img, cmap=mpplot.cm.bone_r) pose_raw = self.yanker(poses_h5, resce_h5, self.caminfo) args.data_draw.draw_pose2d( ax, thedata, args.data_ops.raw_to_2d(pose_raw, thedata) ) rects = cube.proj_rects_3( args.data_ops.raw_to_2d, self.caminfo ) for ii, rect in enumerate(rects): rect.draw(ax, colors[ii]) fig.tight_layout() mpplot.savefig(os.path.join( self.predict_dir, 'draw_{}_{}.png'.format(self.name_desc, img_id))) if self.args.show_draw: mpplot.show() print('[{}] drawing image #{:d} - done.'.format( self.name_desc, img_id))
def draw_raw3d(cls, thedata, img, pose_raw): cube = iso_cube() cube.build(pose_raw) # draw full image fig_size = (2 * 6, 6) fig = mpplot.figure(figsize=fig_size) points3 = dataops.img_to_raw(img, thedata) numpts = points3.shape[0] if 1000 < numpts: samid = np.random.choice(numpts, 1000, replace=False) points3_sam = points3[samid, :] else: points3_sam = points3 ax = fig.add_subplot(1, 2, 1, projection='3d') ax.scatter(points3_sam[:, 0], points3_sam[:, 1], points3_sam[:, 2], color=Color('lightsteelblue').rgb) ax.view_init(azim=-90, elev=-75) ax.set_zlabel('depth (mm)', labelpad=15) cls.draw_raw3d_pose(ax, thedata, pose_raw) corners = cube.get_corners() corners = cube.transform_add_center(corners) cube.draw_wire(corners) # draw cropped region ax = fig.add_subplot(1, 2, 2, projection='3d') points3_trans = cube.pick(points3) points3_trans = cube.transform_to_center(points3_trans) numpts = points3_trans.shape[0] if 1000 < numpts: points3_sam = points3_trans[ np.random.choice(numpts, 1000, replace=False), :] else: points3_sam = points3_trans points3_sam = cube.transform_to_center(points3_sam) pose_trans = cube.transform_to_center(pose_raw) ax.scatter(points3_sam[:, 0], points3_sam[:, 1], points3_sam[:, 2], color=Color('lightsteelblue').rgb) cls.draw_raw3d_pose(ax, thedata, pose_trans) corners = cube.get_corners() cube.draw_wire(corners) ax.view_init(azim=-120, elev=-150) mpplot.tight_layout() mpplot.show() # draw projected image fig_size = (3 * 5, 5) mpplot.subplots(nrows=1, ncols=3, figsize=fig_size) for spi in range(3): ax = mpplot.subplot(1, 3, spi + 1) coord, depth = cube.project_ortho(points3_trans, roll=spi) img = cube.print_image(coord, depth, thedata.crop_size) pose2d, _ = cube.project_ortho(pose_trans, roll=spi, sort=False) cls.draw_pose2d(ax, thedata, pose2d) ax.imshow(img, cmap=mpplot.cm.bone_r) ax.axis('off') mpplot.tight_layout() mpplot.show()
def crop_resize(cls, img, pose_raw, caminfo): # cube.build(pose_raw) cube = iso_cube( (np.max(pose_raw, axis=0) + np.min(pose_raw, axis=0)) / 2, caminfo.region_size) img_crop = cls.to_crop2(img, cube, caminfo) resce = cube.dump() return img_crop, resce
def crop_resize_pca(cls, img, pose_raw, caminfo, sort=False): # return np.zeros((caminfo.crop_size, caminfo.crop_size)), np.ones(4) cube = iso_cube( (np.max(pose_raw, axis=0) + np.min(pose_raw, axis=0)) / 2, caminfo.region_size) img_clean = cls.to_clean(img, cube, caminfo, sort=sort) resce = cube.dump() return img_clean, resce
def simp_crop(self, dimg): caminfo = self.caminfo dimg_f = cv2.bilateralFilter(dimg.astype(np.float32), 5, 30, 30) dlist = dimg_f.ravel() if 10 > len(dlist): self.estr = "hand out of detection range" print(self.estr) self.tracker.clear() return False dpart10 = np.partition(dlist[caminfo.z_range[0] + 0.01 < dlist], 10) z0 = dpart10[9] # print(dlist.shape, np.min(dlist), np.max(dlist)) # print("10th closest point: {}".format(z0)) # 193.10437004605774 if z0 > caminfo.crop_range: self.estr = "hand out of detection range" print(self.estr) self.tracker.clear() return False zrs = z0 + caminfo.region_size in_id = np.where(np.logical_and(z0 - 0.01 < dlist, dlist < zrs)) if 10 > len(in_id[0]): self.estr = "not enough points in range" print(self.estr) self.tracker.clear() return False xin, yin = np.unravel_index(in_id, dimg_f.shape) ## FetchHands17!! { # p2z = np.vstack((yin, xin, dlist[in_id])).T ## } ## live stream { p2z = np.vstack((xin, yin, dlist[in_id])).T ## } p3d = self.args.data_ops.d2z_to_raw(p2z, self.caminfo) ## FetchHands17!! { # cube = iso_cube() # cube.extent_center(p3d) # FetchHands17!! # cube.sidelen = self.caminfo.region_size ## } ## find center { # cen = self.cen_ext.simple_mean(p3d) cen = self.cen_ext.shape_prior(p3d) if cen is False: self.estr = "center not found" print(self.estr) self.tracker.clear() return False cen_m = self.tracker.update(cen) # cen_m = cen print(cen, cen_m) if cen_m is False: self.estr = "lost track" print(self.estr) self.tracker.clear() return False cube = iso_cube(cen_m, self.caminfo.region_size) ## } # # print(cube.dump()) # [120.0000 -158.0551 -116.6658 240.0000] return cube
def convert_output(self, pred_val, args, caminfo): pred_val = pred_val.flatten() halflen = self.crop_range centre = np.append( pred_val[:2] * halflen, pred_val[2] * halflen + halflen, ) cube = iso_cube(centre, self.region_size) return cube
def prow_clean(args, thedata, mode, batch_data): bi, index, resce = \ args[0], args[1], args[2] img_name = dataio.index2imagename(index) img = dataio.read_image(thedata.images_join(img_name, mode)) cube = iso_cube() cube.load(resce) img_clean = dataops.to_clean(img, cube, thedata) batch_data[bi, ...] = img_clean
def test_camera(cap): # test the camera projection: center should align with the image dimension cube = iso_cube(np.array([0, 0, 400]), 120) rects = cube.proj_rects_3( cap.args.data_ops.raw_to_2d, cap.caminfo ) np.set_printoptions(formatter={'float': '{:6.4f}'.format}) for ii, rect in enumerate(rects): rect.show_dims()
def _draw_image_pose(self, ax, frame, poses, resce, caminfo): cube = iso_cube() cube.load(resce) ax.imshow(frame, cmap=mpplot.cm.bone_r) pose2d, _ = cube.raw_to_unit(poses) pose2d *= caminfo.crop_size self.args.data_draw.draw_pose2d( ax, caminfo, pose2d, )
def draw_prediction(self, thedata, args): import linecache import re frame_id = np.random.randint( 1, high=sum(1 for _ in open(self.predict_file, 'r'))) with h5py.File(self.appen_test, 'r') as h5file: img_id = h5file['index'][frame_id, 0] frame_h5 = np.squeeze(h5file['frame'][frame_id, ...], -1) # poses_h5 = h5file['poses'][frame_id, ...] resce_h5 = h5file['resce'][frame_id, ...] frame_h5 = args.data_ops.frame_size_localizer( frame_h5, self.caminfo) print('[{}] drawing image #{:d} ...'.format(self.name_desc, img_id)) colors = [Color('orange').rgb, Color('red').rgb, Color('lime').rgb] ax = mpplot.subplot(1, 2, 1) annot_line = args.data_io.get_line(thedata.annotation_train, img_id) img_name, _ = args.data_io.parse_line_annot(annot_line) img = args.data_io.read_image(os.path.join(self.image_dir, img_name)) ax.imshow(img, cmap=mpplot.cm.bone_r) resce3 = resce_h5[0:4] cube = iso_cube() cube.load(resce3) cube.show_dims() rects = cube.proj_rects_3(args.data_ops.raw_to_2d, self.caminfo) for ii, rect in enumerate(rects): rect.draw(ax, colors[ii]) mpplot.gca().set_title('Ground truth') ax = mpplot.subplot(1, 2, 2) img = frame_h5 ax.imshow(img, cmap=mpplot.cm.bone_r) line_pred = linecache.getline(self.predict_file, frame_id) pred_list = re.split(r'\s+', line_pred.strip()) centre = np.array([float(i) for i in pred_list[1:4]]) cube = iso_cube(centre, self.region_size) cube.show_dims() rects = cube.proj_rects_3(args.data_ops.raw_to_2d, self.caminfo) for ii, rect in enumerate(rects): rect.draw(ax, colors[ii]) mpplot.gca().set_title('Prediction')
def update(i): print("==== Frame: ", i, "====") canvas = self.canvas camframes = self.cam.provide() if camframes is None: return depth_image = camframes.depth color_image = camframes.color canvas.ims[0].set_data(depth_image) canvas.ims[1].set_data(color_image) cube = iso_cube(np.array([0, 0, 400]), 120) # cube=iso_cube(np.array([-200, 20, 400]), 120) self.show_results(canvas, cube)
def prow_index(args, thedata, batch_data): bi, line = \ args[0], args[1] img_name, pose_raw = dataio.parse_line_annot(line) pose2d = dataops.raw_to_2d(pose_raw, thedata) if (0 > np.min(pose2d)) or (0 > np.min(thedata.image_size - pose2d)): return index = dataio.imagename2index(img_name) cube = iso_cube((np.max(pose_raw, axis=0) + np.min(pose_raw, axis=0)) / 2, thedata.region_size) batch_data['valid'][bi] = True batch_data['index'][bi, ...] = index batch_data['poses'][bi, ...] = pose_raw batch_data['resce'][bi, ...] = cube.dump()
def prow_index(args, thedata, mode, batch_data): bi, index, poses = \ args[0], args[1], args[2] pose_raw = poses.reshape(-1, 3) # pose_raw[:, [0, 1]] = pose_raw[:, [1, 0]] pose2d = dataops.raw_to_2d(pose_raw, thedata) if (0 > np.min(pose2d)) or (0 > np.min(thedata.image_size - pose2d)): return cube = iso_cube((np.max(pose_raw, axis=0) + np.min(pose_raw, axis=0)) / 2, thedata.region_size) batch_data['valid'][bi] = True batch_data['index'][bi, ...] = index batch_data['poses'][bi, ...] = pose_raw batch_data['resce'][bi, ...] = cube.dump()
def voxelize_depth(cls, img, pose_raw, step, anchor_num, caminfo): halflen = caminfo.crop_range points3 = cls.img_to_raw(img, caminfo, halflen) grid = regu_grid(np.array([-halflen, -halflen, caminfo.z_range[0]]), step, halflen * 2 / step) pcnt = grid.fill(points3) cube = iso_cube( (np.max(pose_raw, axis=0) + np.min(pose_raw, axis=0)) / 2, caminfo.region_size) grid.step = anchor_num grid.cellen = halflen * 2 / anchor_num anchors = grid.prow_anchor_single(cube.cen, caminfo.region_size) cubecen = grid.fill([cube.cen]) cubecen_anchors = np.append(cubecen.flatten(), anchors) resce = cube.dump() # mpplot = import_module('matplotlib.pyplot') # print(np.histogram(pcnt)) # grid.show_dims() # cube.show_dims() # index = np.array(np.unravel_index(np.argmax(cubecen), cubecen.shape)) # print(index) # print(grid.yank_anchor_single( # index, # anchors # )) # ax = mpplot.subplot(projection='3d') # numpts = points3.shape[0] # if 1000 < numpts: # samid = np.random.choice(numpts, 1000, replace=False) # points3_sam = points3[samid, :] # else: # points3_sam = points3 # ax.scatter( # points3_sam[:, 0], points3_sam[:, 1], points3_sam[:, 2]) # ax.view_init(azim=-90, elev=-75) # ax.set_zlabel('depth (mm)', labelpad=15) # corners = cube.get_corners() # iso_cube.draw_cube_wire(ax, corners) # from mayavi import mlab # mlab.figure(size=(800, 800)) # mlab.pipeline.volume(mlab.pipeline.scalar_field(pcnt)) # mlab.pipeline.image_plane_widget( # mlab.pipeline.scalar_field(pcnt), # plane_orientation='z_axes', # slice_index=halflen) # np.set_printoptions(precision=4) # mlab.outline() # mpplot.show() return pcnt, cubecen_anchors, resce