def __call__(self, img_input): t = time.time() img_input = np.transpose( img_input, axes=[1, 0, 2]).copy() if self.T else img_input img_batch = self._gen_input_batch(img_input, self._box_size, self.scales) # inference hm, xm, ym, zm = self.sess.run( [self.heatmap, self.x_heatmap, self.y_heatmap, self.z_heatmap], {self.input_crops: img_batch}) # average scale outputs hm_size = self._box_size // self._hm_factor hm_avg = np.zeros((hm_size, hm_size, self._joints_num)) xm_avg = np.zeros((hm_size, hm_size, self._joints_num)) ym_avg = np.zeros((hm_size, hm_size, self._joints_num)) zm_avg = np.zeros((hm_size, hm_size, self._joints_num)) for i in range(len(self.scales)): rescale = 1.0 / self.scales[i] scaled_hm = utils.img_scale(hm[i, :, :, :], rescale) scaled_x_hm = utils.img_scale(xm[i, :, :, :], rescale) scaled_y_hm = utils.img_scale(ym[i, :, :, :], rescale) scaled_z_hm = utils.img_scale(zm[i, :, :, :], rescale) mid = [scaled_hm.shape[0] // 2, scaled_hm.shape[1] // 2] hm_avg += scaled_hm[mid[0] - hm_size // 2:mid[0] + hm_size // 2, mid[1] - hm_size // 2:mid[1] + hm_size // 2, :] xm_avg += scaled_x_hm[mid[0] - hm_size // 2:mid[0] + hm_size // 2, mid[1] - hm_size // 2:mid[1] + hm_size // 2, :] ym_avg += scaled_y_hm[mid[0] - hm_size // 2:mid[0] + hm_size // 2, mid[1] - hm_size // 2:mid[1] + hm_size // 2, :] zm_avg += scaled_z_hm[mid[0] - hm_size // 2:mid[0] + hm_size // 2, mid[1] - hm_size // 2:mid[1] + hm_size // 2, :] hm_avg /= len(self.scales) xm_avg /= len(self.scales) ym_avg /= len(self.scales) zm_avg /= len(self.scales) joints_2d = utils.extract_2d_joints_from_heatmap( hm_avg, self._box_size, self._hm_factor) joints_3d = utils.extract_3d_joints_from_heatmap( joints_2d, xm_avg, ym_avg, zm_avg, self._box_size, self._hm_factor) joints_2d, joints_3d = self._joint_filter(joints_2d, joints_3d) # if self.T: # joints_2d = joints_2d[:, ::-1] # joints_3d = joints_3d[:, [1, 0, 2]] print('FPS: {:>2.2f}'.format(1 / (time.time() - t))) if self.plot: # 2d plotting frame_square = utils.img_scale_squareify(img_input, self._box_size) frame_square = utils.draw_limbs_2d(frame_square, joints_2d, self._joint_parents) cv2.imshow('2D Prediction', frame_square) # 3d plotting self.imshow_3d(self.ax_3d, joints_3d, self._joint_parents) return joints_2d, joints_3d
def __call__(self, img_input): t0 = time.time() img_batch, scaler, [offset_x, offset_y ] = self.gen_input_batch(img_input, self.box_size, self.scales) hm, xm, ym, zm = self.sess.run( [self.heatmap, self.x_heatmap, self.y_heatmap, self.z_heatmap], {self.input_crops: img_batch}) # averaging the outputs with different scales hm_size = self.box_size // self.hm_factor hm_avg = np.zeros((hm_size, hm_size, self.joints_sum)) xm_avg = np.zeros((hm_size, hm_size, self.joints_sum)) ym_avg = np.zeros((hm_size, hm_size, self.joints_sum)) zm_avg = np.zeros((hm_size, hm_size, self.joints_sum)) for i in range(len(self.scales)): rescale = 1.0 / self.scales[i] scaled_hm = utils.img_scale(hm[i, :, :, :], rescale) scaled_x_hm = utils.img_scale(xm[i, :, :, :], rescale) scaled_y_hm = utils.img_scale(ym[i, :, :, :], rescale) scaled_z_hm = utils.img_scale(zm[i, :, :, :], rescale) mid = [scaled_hm.shape[0] // 2, scaled_hm.shape[1] // 2] hm_avg += scaled_hm[mid[0] - hm_size // 2:mid[0] + hm_size // 2, mid[1] - hm_size // 2:mid[1] + hm_size // 2, :] xm_avg += scaled_x_hm[mid[0] - hm_size // 2:mid[0] + hm_size // 2, mid[1] - hm_size // 2:mid[1] + hm_size // 2, :] ym_avg += scaled_y_hm[mid[0] - hm_size // 2:mid[0] + hm_size // 2, mid[1] - hm_size // 2:mid[1] + hm_size // 2, :] zm_avg += scaled_z_hm[mid[0] - hm_size // 2:mid[0] + hm_size // 2, mid[1] - hm_size // 2:mid[1] + hm_size // 2, :] hm_avg /= len(self.scales) xm_avg /= len(self.scales) ym_avg /= len(self.scales) zm_avg /= len(self.scales) # joints_2d are in box size scale joints_2d = utils.extract_2d_joints(hm_avg, self.box_size, self.hm_factor) joints_2d = self.joint_filter(joints_2d, dim=2) joints_3d = utils.extract_3d_joints(joints_2d, xm_avg, ym_avg, zm_avg, self.hm_factor) joints_3d = self.joint_filter(joints_3d, dim=3) # rescale joints_2d to input image scale joints_2d[:, 0] = (joints_2d[:, 0] - offset_y) / scaler joints_2d[:, 1] = (joints_2d[:, 1] - offset_x) / scaler print('FPS: {:>2.2f}'.format(1 / (time.time() - t0))) return joints_2d, joints_3d
def __call__(self, img_input): t = time.time() img_batch = self._gen_input_batch(img_input, self._box_size, self.scales) # inference hm, xm, ym, zm = self.sess.run( [self.heatmap, self.x_heatmap, self.y_heatmap, self.z_heatmap], {self.input_crops: img_batch}) # average scale outputs hm_size = self._box_size // self._hm_factor hm_avg = np.zeros((hm_size, hm_size, self._joints_num)) xm_avg = np.zeros((hm_size, hm_size, self._joints_num)) ym_avg = np.zeros((hm_size, hm_size, self._joints_num)) zm_avg = np.zeros((hm_size, hm_size, self._joints_num)) for i in range(len(self.scales)): rescale = 1.0 / self.scales[i] scaled_hm = utils.img_scale(hm[i, :, :, :], rescale) scaled_x_hm = utils.img_scale(xm[i, :, :, :], rescale) scaled_y_hm = utils.img_scale(ym[i, :, :, :], rescale) scaled_z_hm = utils.img_scale(zm[i, :, :, :], rescale) mid = [scaled_hm.shape[0] // 2, scaled_hm.shape[1] // 2] hm_avg += scaled_hm[mid[0] - hm_size // 2:mid[0] + hm_size // 2, mid[1] - hm_size // 2:mid[1] + hm_size // 2, :] xm_avg += scaled_x_hm[mid[0] - hm_size // 2:mid[0] + hm_size // 2, mid[1] - hm_size // 2:mid[1] + hm_size // 2, :] ym_avg += scaled_y_hm[mid[0] - hm_size // 2:mid[0] + hm_size // 2, mid[1] - hm_size // 2:mid[1] + hm_size // 2, :] zm_avg += scaled_z_hm[mid[0] - hm_size // 2:mid[0] + hm_size // 2, mid[1] - hm_size // 2:mid[1] + hm_size // 2, :] hm_avg /= len(self.scales) xm_avg /= len(self.scales) ym_avg /= len(self.scales) zm_avg /= len(self.scales) joints_2d = utils.extract_2d_joints_from_heatmap( hm_avg, self._box_size, self._hm_factor) joints_3d = utils.extract_3d_joints_from_heatmap( joints_2d, xm_avg, ym_avg, zm_avg, self._box_size, self._hm_factor) joints_2d, joints_3d = self._joint_filter(joints_2d, joints_3d) print('FPS: {:>2.2f}'.format(1 / (time.time() - t))) return joints_2d, joints_3d
def _run_net(self): # inference hm, xm, ym, zm = self.sess.run( [self.heatmap, self.x_heatmap, self.y_heatmap, self.z_heatmap], {self.input_crops: self.input_batch}) # average scale outputs hm_size = self.box_size // self.hm_factor hm_avg = np.zeros((hm_size, hm_size, self.joints_num)) xm_avg = np.zeros((hm_size, hm_size, self.joints_num)) ym_avg = np.zeros((hm_size, hm_size, self.joints_num)) zm_avg = np.zeros((hm_size, hm_size, self.joints_num)) for i in range(len(self.scales)): rescale = 1.0 / self.scales[i] scaled_hm = utils.img_scale(hm[i, :, :, :], rescale) scaled_x_hm = utils.img_scale(xm[i, :, :, :], rescale) scaled_y_hm = utils.img_scale(ym[i, :, :, :], rescale) scaled_z_hm = utils.img_scale(zm[i, :, :, :], rescale) mid = [scaled_hm.shape[0] // 2, scaled_hm.shape[1] // 2] hm_avg += scaled_hm[mid[0] - hm_size // 2:mid[0] + hm_size // 2, mid[1] - hm_size // 2:mid[1] + hm_size // 2, :] xm_avg += scaled_x_hm[mid[0] - hm_size // 2:mid[0] + hm_size // 2, mid[1] - hm_size // 2:mid[1] + hm_size // 2, :] ym_avg += scaled_y_hm[mid[0] - hm_size // 2:mid[0] + hm_size // 2, mid[1] - hm_size // 2:mid[1] + hm_size // 2, :] zm_avg += scaled_z_hm[mid[0] - hm_size // 2:mid[0] + hm_size // 2, mid[1] - hm_size // 2:mid[1] + hm_size // 2, :] hm_avg /= len(self.scales) xm_avg /= len(self.scales) ym_avg /= len(self.scales) zm_avg /= len(self.scales) self.joints_2d = utils.extract_2d_joints_from_heatmap( hm_avg, self.box_size, self.hm_factor) self.joints_3d = utils.extract_3d_joints_from_heatmap( self.joints_2d, xm_avg, ym_avg, zm_avg, self.box_size, self.hm_factor)
def addSliders(self): self.sliders = [] self.sfn = { 'shift_x': lambda i,v: utils.img_shift_x(i, v*28), 'shift_y': lambda i,v: utils.img_shift_y(i, v*28), 'blur': lambda i,v: utils.img_blur(i, v*10), 'rotation': lambda i,v: utils.img_rotate(i, v*180.), 'scale': lambda i,v: utils.img_scale(i, 1+v), 'sindisp_x': lambda i,v: utils.img_sindisp_x(i, v*10), 'sindisp_y': lambda i,v: utils.img_sindisp_y(i, v*10), } self.svalues = { k:0 for k in self.sfn.keys() } for k, sfn in self.sfn.iteritems(): s = QSlider(self.verticalLayoutWidget) s.setOrientation(QtCore.Qt.Horizontal) s.setSliderPosition(50) self.sliders.append(s) fn = lambda v,k=k,sfn=sfn: self.onSlider(k, sfn, v/50. - 1.) QtCore.QObject.connect(s, QtCore.SIGNAL('valueChanged(int)'), fn) self.verticalLayout_3.addWidget(QLabel(k)) self.verticalLayout_3.addWidget(s)