def __init__(self, use_gpu=False): actions = data_utils.define_actions(FLAGS.action) rcams = cameras.load_cameras(FLAGS.cameras_path, SUBJECT_IDS) train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.read_2d_predictions( actions, FLAGS.data_dir) train_set_3d, test_set_3d, data_mean_3d, data_std_3d, dim_to_ignore_3d, dim_to_use_3d, train_root_positions, test_root_positions = data_utils.read_3d_data( actions, FLAGS.data_dir, FLAGS.camera_frame, rcams, FLAGS.predict_14) self.train_set_2d = train_set_2d self.test_set_2d = test_set_2d self.data_mean_2d = data_mean_2d self.data_std_2d = data_std_2d self.dim_to_use_2d = dim_to_use_2d self.dim_to_ignore_2d = dim_to_ignore_2d self.train_set_3d = train_set_3d self.test_set_3d = test_set_3d self.data_mean_3d = data_mean_3d self.data_std_3d = data_std_3d self.dim_to_use_3d = dim_to_use_3d self.dim_to_ignore_3d = dim_to_ignore_3d device_count = {"GPU": 1} if use_gpu else {"GPU": 0} self.persistent_sess = tf.Session(config=tf.ConfigProto( device_count=device_count, allow_soft_placement=True)) with self.persistent_sess.as_default(): self.graph = tf.get_default_graph() self.model = create_model(self.persistent_sess, actions, batch_size)
def main(_): #ABS_DIR = os.path.abspath('.') posf = open(pose_output_dir, 'w') #smoothedf = open(ABS_DIR + '/tmp/smoothed.txt', 'w') smoothed = read_openpose_json() plt.figure(2) smooth_curves_plot = show_anim_curves(smoothed, plt) pngName = 'gif_output/smooth_plot.png' smooth_curves_plot.savefig(pngName) logger.info('writing gif_output/smooth_plot.png') if FLAGS.interpolation: logger.info("start interpolation") framerange = len( smoothed.keys() ) joint_rows = 36 array = np.concatenate(list(smoothed.values())) array_reshaped = np.reshape(array, (framerange, joint_rows) ) print(array_reshaped[0,:]) arm = [4,5,6,7,8,9,10,11] multiplier = FLAGS.multiplier multiplier_inv = 1/multiplier out_array = np.array([]) for row in range(joint_rows): x = [] for frame in range(framerange): x.append( array_reshaped[frame, row] ) frame = range( framerange ) frame_resampled = np.arange(0, framerange, multiplier) spl = UnivariateSpline(frame, x, k=3) #relative smooth factor based on jnt anim curve min_x, max_x = min(x), max(x) smooth_fac = max_x - min_x if row in arm: smooth_resamp = 1 else: smooth_resamp = 75 smooth_fac = smooth_fac * smooth_resamp spl.set_smoothing_factor( float(smooth_fac) ) xnew = spl(frame_resampled) out_array = np.append(out_array, xnew) logger.info("done interpolating. reshaping {0} frames, please wait!!".format(framerange)) a = np.array([]) for frame in range( int( framerange * multiplier_inv ) ): jnt_array = [] for jnt in range(joint_rows): jnt_array.append( out_array[ jnt * int(framerange * multiplier_inv) + frame] ) a = np.append(a, jnt_array) a = np.reshape(a, (int(framerange * multiplier_inv), joint_rows)) out_array = a interpolate_smoothed = {} for frame in range( int(framerange * multiplier_inv) ): interpolate_smoothed[frame] = list( out_array[frame] ) plt.figure(3) smoothed = interpolate_smoothed interpolate_curves_plot = show_anim_curves(smoothed, plt) pngName = 'gif_output/interpolate_{0}.png'.format(smooth_resamp) interpolate_curves_plot.savefig(pngName) logger.info('writing gif_output/interpolate_plot.png') enc_in = np.zeros((1, 64)) enc_in[0] = [0 for i in range(64)] actions = data_utils.define_actions(FLAGS.action) SUBJECT_IDS = [1, 5, 6, 7, 8, 9, 11] rcams = cameras.load_cameras(FLAGS.cameras_path, SUBJECT_IDS) train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.read_2d_predictions( actions, FLAGS.data_dir) train_set_3d, test_set_3d, data_mean_3d, data_std_3d, dim_to_ignore_3d, dim_to_use_3d, train_root_positions, test_root_positions = data_utils.read_3d_data( actions, FLAGS.data_dir, FLAGS.camera_frame, rcams, FLAGS.predict_14) device_count = {"GPU": 1} png_lib = [] with tf.Session(config=tf.ConfigProto( device_count=device_count, allow_soft_placement=True)) as sess: #plt.figure(3) batch_size = 128 model = create_model(sess, actions, batch_size) iter_range = len(smoothed.keys()) for n, (frame, xy) in enumerate(smoothed.items()): logger.info("calc frame {0}/{1}".format(frame, iter_range)) # map list into np array joints_array = np.zeros((1, 36)) joints_array[0] = [0 for i in range(36)] for o in range(len(joints_array[0])): #feed array with xy array joints_array[0][o] = xy[o] _data = joints_array[0] #smoothedf.write(' '.join(map(str, _data))) #smoothedf.write("\n") # mapping all body parts or 3d-pose-baseline format for i in range(len(order)): for j in range(2): # create encoder input enc_in[0][order[i] * 2 + j] = _data[i * 2 + j] for j in range(2): # Hip enc_in[0][0 * 2 + j] = (enc_in[0][1 * 2 + j] + enc_in[0][6 * 2 + j]) / 2 # Neck/Nose enc_in[0][14 * 2 + j] = (enc_in[0][15 * 2 + j] + enc_in[0][12 * 2 + j]) / 2 # Thorax enc_in[0][13 * 2 + j] = 2 * enc_in[0][12 * 2 + j] - enc_in[0][14 * 2 + j] # set spine spine_x = enc_in[0][24] spine_y = enc_in[0][25] enc_in = enc_in[:, dim_to_use_2d] mu = data_mean_2d[dim_to_use_2d] stddev = data_std_2d[dim_to_use_2d] enc_in = np.divide((enc_in - mu), stddev) dp = 1.0 dec_out = np.zeros((1, 48)) dec_out[0] = [0 for i in range(48)] _, _, poses3d = model.step(sess, enc_in, dec_out, dp, isTraining=False) all_poses_3d = [] enc_in = data_utils.unNormalizeData(enc_in, data_mean_2d, data_std_2d, dim_to_ignore_2d) poses3d = data_utils.unNormalizeData(poses3d, data_mean_3d, data_std_3d, dim_to_ignore_3d) gs1 = gridspec.GridSpec(1, 1) gs1.update(wspace=-0.00, hspace=0.05) # set the spacing between axes. plt.axis('off') all_poses_3d.append( poses3d ) enc_in, poses3d = map( np.vstack, [enc_in, all_poses_3d] ) subplot_idx, exidx = 1, 1 _max = 0 _min = 10000 for i in range(poses3d.shape[0]): for j in range(32): tmp = poses3d[i][j * 3 + 2] poses3d[i][j * 3 + 2] = poses3d[i][j * 3 + 1] poses3d[i][j * 3 + 1] = tmp if poses3d[i][j * 3 + 2] > _max: _max = poses3d[i][j * 3 + 2] if poses3d[i][j * 3 + 2] < _min: _min = poses3d[i][j * 3 + 2] for i in range(poses3d.shape[0]): for j in range(32): poses3d[i][j * 3 + 2] = _max - poses3d[i][j * 3 + 2] + _min poses3d[i][j * 3] += (spine_x - 630) poses3d[i][j * 3 + 2] += (500 - spine_y) # Plot 3d predictions ax = plt.subplot(gs1[subplot_idx - 1], projection='3d') ax.view_init(18, -70) if np.min(poses3d) < -1000: try: poses3d = before_pose except: pass p3d = poses3d #viz.show3Dpose(p3d, ax, lcolor="#9b59b6", rcolor="#2ecc71") # pngName = 'png/pose_frame_{0}.png'.format(str(frame).zfill(12)) # plt.savefig(pngName) # if FLAGS.write_gif: # png_lib.append(imageio.imread(pngName)) before_pose = poses3d write_pos_data(poses3d, ax, posf) posf.close()
def main(_): smoothed = read_openpose_json() plt.figure(2) smooth_curves_plot = show_anim_curves(smoothed, plt) #return pngName = 'gif_output/smooth_plot.png' smooth_curves_plot.savefig(pngName) logger.info('writing gif_output/smooth_plot.png') if FLAGS.interpolation: logger.info("start interpolation") framerange = len( smoothed.keys() ) joint_rows = 36 array = np.concatenate(list(smoothed.values())) array_reshaped = np.reshape(array, (framerange, joint_rows) ) multiplier = FLAGS.multiplier multiplier_inv = 1/multiplier out_array = np.array([]) for row in range(joint_rows): x = [] for frame in range(framerange): x.append( array_reshaped[frame, row] ) frame = range( framerange ) frame_resampled = np.arange(0, framerange, multiplier) spl = UnivariateSpline(frame, x, k=3) #relative smooth factor based on jnt anim curve min_x, max_x = min(x), max(x) smooth_fac = max_x - min_x smooth_resamp = 125 smooth_fac = smooth_fac * smooth_resamp spl.set_smoothing_factor( float(smooth_fac) ) xnew = spl(frame_resampled) out_array = np.append(out_array, xnew) logger.info("done interpolating. reshaping {0} frames, please wait!!".format(framerange)) a = np.array([]) for frame in range( int( framerange * multiplier_inv ) ): jnt_array = [] for jnt in range(joint_rows): jnt_array.append( out_array[ jnt * int(framerange * multiplier_inv) + frame] ) a = np.append(a, jnt_array) a = np.reshape(a, (int(framerange * multiplier_inv), joint_rows)) out_array = a interpolate_smoothed = {} for frame in range( int(framerange * multiplier_inv) ): interpolate_smoothed[frame] = list( out_array[frame] ) plt.figure(3) smoothed = interpolate_smoothed interpolate_curves_plot = show_anim_curves(smoothed, plt) pngName = 'gif_output/interpolate_{0}.png'.format(smooth_resamp) interpolate_curves_plot.savefig(pngName) logger.info('writing gif_output/interpolate_plot.png') enc_in = np.zeros((1, 64)) enc_in[0] = [0 for i in range(64)] actions = data_utils.define_actions(FLAGS.action) SUBJECT_IDS = [1, 5, 6, 7, 8, 9, 11] rcams = cameras.load_cameras(FLAGS.cameras_path, SUBJECT_IDS) train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.read_2d_predictions( actions, FLAGS.data_dir) train_set_3d, test_set_3d, data_mean_3d, data_std_3d, dim_to_ignore_3d, dim_to_use_3d, train_root_positions, test_root_positions = data_utils.read_3d_data( actions, FLAGS.data_dir, FLAGS.camera_frame, rcams, FLAGS.predict_14) device_count = {"GPU": 1} png_lib = [] before_pose = None with tf.Session(config=tf.ConfigProto( device_count=device_count, allow_soft_placement=True)) as sess: #plt.figure(3) batch_size = 128 model = create_model(sess, actions, batch_size) iter_range = len(smoothed.keys()) export_units = {} twod_export_units = {} for n, (frame, xy) in enumerate(smoothed.items()): logger.info("calc frame {0}/{1}".format(frame, iter_range)) # map list into np array joints_array = np.zeros((1, 36)) joints_array[0] = [0 for i in range(36)] for o in range(len(joints_array[0])): #feed array with xy array joints_array[0][o] = xy[o] twod_export_units[frame]={} for abs_b, __n in enumerate(range(0, len(xy),2)): twod_export_units[frame][abs_b] = {"translate": [xy[__n],xy[__n+1]]} _data = joints_array[0] # mapping all body parts or 3d-pose-baseline format for i in range(len(order)): for j in range(2): # create encoder input enc_in[0][order[i] * 2 + j] = _data[i * 2 + j] for j in range(2): # Hip enc_in[0][0 * 2 + j] = (enc_in[0][1 * 2 + j] + enc_in[0][6 * 2 + j]) / 2 # Neck/Nose enc_in[0][14 * 2 + j] = (enc_in[0][15 * 2 + j] + enc_in[0][12 * 2 + j]) / 2 # Thorax enc_in[0][13 * 2 + j] = 2 * enc_in[0][12 * 2 + j] - enc_in[0][14 * 2 + j] # set spine spine_x = enc_in[0][24] spine_y = enc_in[0][25] enc_in = enc_in[:, dim_to_use_2d] mu = data_mean_2d[dim_to_use_2d] stddev = data_std_2d[dim_to_use_2d] enc_in = np.divide((enc_in - mu), stddev) dp = 1.0 dec_out = np.zeros((1, 48)) dec_out[0] = [0 for i in range(48)] _, _, poses3d = model.step(sess, enc_in, dec_out, dp, isTraining=False) all_poses_3d = [] enc_in = data_utils.unNormalizeData(enc_in, data_mean_2d, data_std_2d, dim_to_ignore_2d) poses3d = data_utils.unNormalizeData(poses3d, data_mean_3d, data_std_3d, dim_to_ignore_3d) gs1 = gridspec.GridSpec(1, 1) gs1.update(wspace=-0.00, hspace=0.05) # set the spacing between axes. plt.axis('off') all_poses_3d.append( poses3d ) enc_in, poses3d = map( np.vstack, [enc_in, all_poses_3d] ) subplot_idx, exidx = 1, 1 _max = 0 _min = 10000 for i in range(poses3d.shape[0]): for j in range(32): tmp = poses3d[i][j * 3 + 2] poses3d[i][j * 3 + 2] = poses3d[i][j * 3 + 1] poses3d[i][j * 3 + 1] = tmp if poses3d[i][j * 3 + 2] > _max: _max = poses3d[i][j * 3 + 2] if poses3d[i][j * 3 + 2] < _min: _min = poses3d[i][j * 3 + 2] for i in range(poses3d.shape[0]): for j in range(32): poses3d[i][j * 3 + 2] = _max - poses3d[i][j * 3 + 2] + _min poses3d[i][j * 3] += (spine_x - 630) poses3d[i][j * 3 + 2] += (500 - spine_y) # Plot 3d predictions ax = plt.subplot(gs1[subplot_idx - 1], projection='3d') ax.view_init(18, -70) if FLAGS.cache_on_fail: if np.min(poses3d) < -1000: poses3d = before_pose p3d = poses3d to_export = poses3d.tolist()[0] x,y,z = [[] for _ in range(3)] for o in range(0, len(to_export), 3): x.append(to_export[o]) y.append(to_export[o+1]) z.append(to_export[o+2]) export_units[frame]={} for jnt_index, (_x, _y, _z) in enumerate(zip(x,y,z)): export_units[frame][jnt_index] = {"translate": [_x, _y, _z]} viz.show3Dpose(p3d, ax, lcolor="#9b59b6", rcolor="#2ecc71") pngName = 'png/pose_frame_{0}.png'.format(str(frame).zfill(12)) plt.savefig(pngName) if FLAGS.write_gif: png_lib.append(imageio.imread(pngName)) if FLAGS.cache_on_fail: before_pose = poses3d if FLAGS.write_gif: if FLAGS.interpolation: #take every frame on gif_fps * multiplier_inv png_lib = np.array([png_lib[png_image] for png_image in range(0,len(png_lib), int(multiplier_inv)) ]) logger.info("creating Gif gif_output/animation.gif, please Wait!") imageio.mimsave('gif_output/animation.gif', png_lib, fps=FLAGS.gif_fps) _out_file = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'maya/3d_data.json') twod_out_file = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'maya/2d_data.json') with open(_out_file, 'w') as outfile: logger.info("exported maya json to {0}".format(_out_file)) json.dump(export_units, outfile) with open(twod_out_file, 'w') as outfile: logger.info("exported maya json to {0}".format(twod_out_file)) json.dump(twod_export_units, outfile) logger.info("Done!".format(pngName))
def main(_): # 出力用日付 now_str = "{0:%Y%m%d_%H%M%S}".format(datetime.datetime.now()) logger.debug("FLAGS.person_idx={0}".format(FLAGS.person_idx)) # 日付+indexディレクトリ作成 subdir = '{0}/{1}_3d_{2}_idx{3:02d}'.format( os.path.dirname(openpose_output_dir), os.path.basename(openpose_output_dir), now_str, FLAGS.person_idx) os.makedirs(subdir) frame3d_dir = "{0}/frame3d".format(subdir) os.makedirs(frame3d_dir) #関節位置情報ファイル posf = open(subdir + '/pos.txt', 'w') #正規化済みOpenpose位置情報ファイル smoothedf = open(subdir + '/smoothed.txt', 'w') idx = FLAGS.person_idx - 1 smoothed = openpose_utils.read_openpose_json(openpose_output_dir, idx, level[FLAGS.verbose] == 3) logger.info("reading and smoothing done. start feeding 3d-pose-baseline") logger.debug(smoothed) plt.figure(2) smooth_curves_plot = show_anim_curves(smoothed, plt) pngName = subdir + '/smooth_plot.png' smooth_curves_plot.savefig(pngName) enc_in = np.zeros((1, 64)) enc_in[0] = [0 for i in range(64)] actions = data_utils.define_actions(FLAGS.action) SUBJECT_IDS = [1, 5, 6, 7, 8, 9, 11] rcams = cameras.load_cameras(FLAGS.cameras_path, SUBJECT_IDS) train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.read_2d_predictions( actions, FLAGS.data_dir) train_set_3d, test_set_3d, data_mean_3d, data_std_3d, dim_to_ignore_3d, dim_to_use_3d, train_root_positions, test_root_positions = data_utils.read_3d_data( actions, FLAGS.data_dir, FLAGS.camera_frame, rcams, FLAGS.predict_14) before_pose = None device_count = {"GPU": 1} png_lib = [] with tf.Session(config=tf.ConfigProto(device_count=device_count, allow_soft_placement=True)) as sess: #plt.figure(3) batch_size = 128 model = create_model(sess, actions, batch_size) for n, (frame, xy) in enumerate(smoothed.items()): logger.info("calc idx {0}, frame {1}".format(idx, frame)) # map list into np array joints_array = np.zeros((1, 36)) joints_array[0] = [0 for i in range(36)] for o in range(len(joints_array[0])): #feed array with xy array joints_array[0][o] = xy[o] _data = joints_array[0] smoothedf.write(' '.join(map(str, _data))) smoothedf.write("\n") # mapping all body parts or 3d-pose-baseline format for i in range(len(order)): for j in range(2): # create encoder input enc_in[0][order[i] * 2 + j] = _data[i * 2 + j] for j in range(2): # Hip enc_in[0][0 * 2 + j] = (enc_in[0][1 * 2 + j] + enc_in[0][6 * 2 + j]) / 2 # Neck/Nose enc_in[0][14 * 2 + j] = (enc_in[0][15 * 2 + j] + enc_in[0][12 * 2 + j]) / 2 # Thorax enc_in[0][13 * 2 + j] = 2 * enc_in[0][12 * 2 + j] - enc_in[0][14 * 2 + j] # set spine spine_x = enc_in[0][24] spine_y = enc_in[0][25] # logger.debug("enc_in - 1") # logger.debug(enc_in) enc_in = enc_in[:, dim_to_use_2d] mu = data_mean_2d[dim_to_use_2d] stddev = data_std_2d[dim_to_use_2d] enc_in = np.divide((enc_in - mu), stddev) dp = 1.0 dec_out = np.zeros((1, 48)) dec_out[0] = [0 for i in range(48)] _, _, poses3d = model.step(sess, enc_in, dec_out, dp, isTraining=False) all_poses_3d = [] enc_in = data_utils.unNormalizeData(enc_in, data_mean_2d, data_std_2d, dim_to_ignore_2d) poses3d = data_utils.unNormalizeData(poses3d, data_mean_3d, data_std_3d, dim_to_ignore_3d) gs1 = gridspec.GridSpec(1, 1) gs1.update(wspace=-0.00, hspace=0.05) # set the spacing between axes. plt.axis('off') all_poses_3d.append(poses3d) enc_in, poses3d = map(np.vstack, [enc_in, all_poses_3d]) subplot_idx, exidx = 1, 1 max = 0 min = 10000 # logger.debug("enc_in - 2") # logger.debug(enc_in) for i in range(poses3d.shape[0]): for j in range(32): tmp = poses3d[i][j * 3 + 2] poses3d[i][j * 3 + 2] = poses3d[i][j * 3 + 1] poses3d[i][j * 3 + 1] = tmp if poses3d[i][j * 3 + 2] > max: max = poses3d[i][j * 3 + 2] if poses3d[i][j * 3 + 2] < min: min = poses3d[i][j * 3 + 2] for i in range(poses3d.shape[0]): for j in range(32): poses3d[i][j * 3 + 2] = max - poses3d[i][j * 3 + 2] + min poses3d[i][j * 3] += (spine_x - 630) poses3d[i][j * 3 + 2] += (500 - spine_y) # Plot 3d predictions ax = plt.subplot(gs1[subplot_idx - 1], projection='3d') ax.view_init(18, 280) logger.debug(np.min(poses3d)) if np.min(poses3d) < -1000 and before_pose is not None: poses3d = before_pose p3d = poses3d # logger.debug("poses3d") # logger.debug(poses3d) if level[FLAGS.verbose] == logging.INFO: viz.show3Dpose(p3d, ax, lcolor="#9b59b6", rcolor="#2ecc71", add_labels=True) # 各フレームの単一視点からのはINFO時のみ pngName = frame3d_dir + '/tmp_{0:012d}.png'.format(frame) plt.savefig(pngName) png_lib.append(imageio.imread(pngName)) before_pose = poses3d # 各フレームの角度別出力はデバッグ時のみ if level[FLAGS.verbose] == logging.DEBUG: for azim in [0, 45, 90, 135, 180, 225, 270, 315, 360]: ax2 = plt.subplot(gs1[subplot_idx - 1], projection='3d') ax2.view_init(18, azim) viz.show3Dpose(p3d, ax2, lcolor="#FF0000", rcolor="#0000FF", add_labels=True) pngName2 = frame3d_dir + '/tmp_{0:012d}_{1:03d}.png'.format( frame, azim) plt.savefig(pngName2) #関節位置情報の出力 write_pos_data(poses3d, ax, posf) posf.close() # INFO時は、アニメーションGIF生成 if level[FLAGS.verbose] == logging.INFO: logger.info( "creating Gif {0}/movie_smoothing.gif, please Wait!".format( subdir)) imageio.mimsave('{0}/movie_smoothing.gif'.format(subdir), png_lib, fps=FLAGS.gif_fps) logger.info("Done!".format(pngName))
def main(_): smoothed = read_openpose_json() logger.info("reading and smoothing done. start feeding 3d-pose-baseline") plt.figure(2) smooth_curves_plot = show_anim_curves(smoothed, plt) pngName = 'png/smooth_plot.png' smooth_curves_plot.savefig(pngName) enc_in = np.zeros((1, 64)) enc_in[0] = [0 for i in range(64)] actions = data_utils.define_actions(FLAGS.action) SUBJECT_IDS = [1, 5, 6, 7, 8, 9, 11] rcams = cameras.load_cameras(FLAGS.cameras_path, SUBJECT_IDS) train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.read_2d_predictions( actions, FLAGS.data_dir) train_set_3d, test_set_3d, data_mean_3d, data_std_3d, dim_to_ignore_3d, dim_to_use_3d, train_root_positions, test_root_positions = data_utils.read_3d_data( actions, FLAGS.data_dir, FLAGS.camera_frame, rcams, FLAGS.predict_14) device_count = {"GPU": 1} png_lib = [] with tf.Session(config=tf.ConfigProto(device_count=device_count, allow_soft_placement=True)) as sess: #plt.figure(3) batch_size = 128 model = create_model(sess, actions, batch_size) for n, (frame, xy) in enumerate(smoothed.items()): logger.info("calc frame {0}".format(frame)) # map list into np array joints_array = np.zeros((1, 36)) joints_array[0] = [0 for i in range(36)] for o in range(len(joints_array[0])): #feed array with xy array joints_array[0][o] = xy[o] _data = joints_array[0] # mapping all body parts or 3d-pose-baseline format for i in range(len(order)): for j in range(2): # create encoder input enc_in[0][order[i] * 2 + j] = _data[i * 2 + j] for j in range(2): # Hip enc_in[0][0 * 2 + j] = (enc_in[0][1 * 2 + j] + enc_in[0][6 * 2 + j]) / 2 # Neck/Nose enc_in[0][14 * 2 + j] = (enc_in[0][15 * 2 + j] + enc_in[0][12 * 2 + j]) / 2 # Thorax enc_in[0][13 * 2 + j] = 2 * enc_in[0][12 * 2 + j] - enc_in[0][14 * 2 + j] # set spine spine_x = enc_in[0][24] spine_y = enc_in[0][25] enc_in = enc_in[:, dim_to_use_2d] mu = data_mean_2d[dim_to_use_2d] stddev = data_std_2d[dim_to_use_2d] enc_in = np.divide((enc_in - mu), stddev) dp = 1.0 dec_out = np.zeros((1, 48)) dec_out[0] = [0 for i in range(48)] _, _, poses3d = model.step(sess, enc_in, dec_out, dp, isTraining=False) all_poses_3d = [] enc_in = data_utils.unNormalizeData(enc_in, data_mean_2d, data_std_2d, dim_to_ignore_2d) poses3d = data_utils.unNormalizeData(poses3d, data_mean_3d, data_std_3d, dim_to_ignore_3d) gs1 = gridspec.GridSpec(1, 1) gs1.update(wspace=-0.00, hspace=0.05) # set the spacing between axes. plt.axis('off') all_poses_3d.append(poses3d) enc_in, poses3d = map(np.vstack, [enc_in, all_poses_3d]) subplot_idx, exidx = 1, 1 max = 0 min = 10000 for i in range(poses3d.shape[0]): for j in range(32): tmp = poses3d[i][j * 3 + 2] poses3d[i][j * 3 + 2] = poses3d[i][j * 3 + 1] poses3d[i][j * 3 + 1] = tmp if poses3d[i][j * 3 + 2] > max: max = poses3d[i][j * 3 + 2] if poses3d[i][j * 3 + 2] < min: min = poses3d[i][j * 3 + 2] for i in range(poses3d.shape[0]): for j in range(32): poses3d[i][j * 3 + 2] = max - poses3d[i][j * 3 + 2] + min poses3d[i][j * 3] += (spine_x - 630) poses3d[i][j * 3 + 2] += (500 - spine_y) # Plot 3d predictions ax = plt.subplot(gs1[subplot_idx - 1], projection='3d') ax.view_init(18, -70) logger.debug(np.min(poses3d)) if np.min(poses3d) < -1000: poses3d = before_pose p3d = poses3d logger.debug(poses3d) viz.show3Dpose(p3d, ax, lcolor="#9b59b6", rcolor="#2ecc71") pngName = 'png/test_{0}.png'.format(str(frame)) plt.savefig(pngName) png_lib.append(imageio.imread(pngName)) before_pose = poses3d logger.info("creating Gif png/movie_smoothing.gif, please Wait!") imageio.mimsave('png/movie_smoothing.gif', png_lib, fps=FLAGS.gif_fps) logger.info("Done!".format(pngName))
def main(_): done = [] enc_in = np.zeros((1, 64)) enc_in[0] = [0 for i in range(64)] actions = data_utils.define_actions(FLAGS.action) SUBJECT_IDS = [1, 5, 6, 7, 8, 9, 11] rcams = cameras.load_cameras(FLAGS.cameras_path, SUBJECT_IDS) train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.read_2d_predictions( actions, FLAGS.data_dir) train_set_3d, test_set_3d, data_mean_3d, data_std_3d, dim_to_ignore_3d, dim_to_use_3d, train_root_positions, test_root_positions = data_utils.read_3d_data( actions, FLAGS.data_dir, FLAGS.camera_frame, rcams, FLAGS.predict_14) device_count = {"GPU": 0} png_lib = [] with tf.Session(config=tf.ConfigProto(device_count=device_count, allow_soft_placement=True)) as sess: #plt.figure(3) batch_size = 128 model = create_model(sess, actions, batch_size) while True: key = cv2.waitKey(1) & 0xFF #logger.info("start reading data") # check for other file types list_of_files = glob.iglob("{0}/*".format( openpose_output_dir)) # You may use iglob in Python3 latest_file = "" try: latest_file = max(list_of_files, key=os.path.getctime) except ValueError: #empthy dir pass if not latest_file: continue try: _file = file_name = latest_file print(latest_file) if not os.path.isfile(_file): raise Exception("No file found!!, {0}".format(_file)) data = json.load(open(_file)) #take first person _data = data["people"][0]["pose_keypoints"] xy = [] #ignore confidence score for o in range(0, len(_data), 3): xy.append(_data[o]) xy.append(_data[o + 1]) frame_indx = re.findall("(\d+)", file_name) frame = int(frame_indx[0]) joints_array = np.zeros((1, 36)) joints_array[0] = [0 for i in range(36)] for o in range(len(joints_array[0])): #feed array with xy array joints_array[0][o] = xy[o] _data = joints_array[0] # mapping all body parts or 3d-pose-baseline format for i in range(len(order)): for j in range(2): # create encoder input enc_in[0][order[i] * 2 + j] = _data[i * 2 + j] for j in range(2): # Hip enc_in[0][0 * 2 + j] = (enc_in[0][1 * 2 + j] + enc_in[0][6 * 2 + j]) / 2 # Neck/Nose enc_in[0][14 * 2 + j] = (enc_in[0][15 * 2 + j] + enc_in[0][12 * 2 + j]) / 2 # Thorax enc_in[0][ 13 * 2 + j] = 2 * enc_in[0][12 * 2 + j] - enc_in[0][14 * 2 + j] # set spine spine_x = enc_in[0][24] spine_y = enc_in[0][25] enc_in = enc_in[:, dim_to_use_2d] mu = data_mean_2d[dim_to_use_2d] stddev = data_std_2d[dim_to_use_2d] enc_in = np.divide((enc_in - mu), stddev) dp = 1.0 dec_out = np.zeros((1, 48)) dec_out[0] = [0 for i in range(48)] _, _, poses3d = model.step(sess, enc_in, dec_out, dp, isTraining=False) all_poses_3d = [] enc_in = data_utils.unNormalizeData(enc_in, data_mean_2d, data_std_2d, dim_to_ignore_2d) poses3d = data_utils.unNormalizeData(poses3d, data_mean_3d, data_std_3d, dim_to_ignore_3d) gs1 = gridspec.GridSpec(1, 1) gs1.update(wspace=-0.00, hspace=0.05) # set the spacing between axes. plt.axis('off') all_poses_3d.append(poses3d) enc_in, poses3d = map(np.vstack, [enc_in, all_poses_3d]) subplot_idx, exidx = 1, 1 _max = 0 _min = 10000 for i in range(poses3d.shape[0]): for j in range(32): tmp = poses3d[i][j * 3 + 2] poses3d[i][j * 3 + 2] = poses3d[i][j * 3 + 1] poses3d[i][j * 3 + 1] = tmp if poses3d[i][j * 3 + 2] > _max: _max = poses3d[i][j * 3 + 2] if poses3d[i][j * 3 + 2] < _min: _min = poses3d[i][j * 3 + 2] for i in range(poses3d.shape[0]): for j in range(32): poses3d[i][j * 3 + 2] = _max - poses3d[i][j * 3 + 2] + _min poses3d[i][j * 3] += (spine_x - 630) poses3d[i][j * 3 + 2] += (500 - spine_y) # Plot 3d predictions ax = plt.subplot(gs1[subplot_idx - 1], projection='3d') ax.view_init(18, -70) logger.debug(np.min(poses3d)) if np.min(poses3d) < -1000 and frame != 0: poses3d = before_pose p3d = poses3d viz.show3Dpose(p3d, ax, lcolor="#9b59b6", rcolor="#2ecc71") before_pose = poses3d pngName = 'png/test_{0}.png'.format(str(frame)) plt.savefig(pngName) #plt.show() img = cv2.imread(pngName, 0) rect_cpy = img.copy() cv2.imshow('3d-pose-baseline', rect_cpy) done.append(file_name) if key == ord('q'): break except Exception as e: print(e) sess.close()
def main(_): global framenum #clear out all old frames os.system("rm png/*") #set done to empty array, it will hold the json files from openpose that we've already processed done = [] #initialize input tensor to 1x64 array of zeroes [[0. 0. 0. ...]] #this is list of numpy vectors to feed as encoder inputs (32 2d coordinates) enc_in = np.zeros((1, 64)) enc_in[0] = [0 for i in range(64)] #actions to run on, default is all actions = data_utils.define_actions(FLAGS.action) #the list of Human3.6m subjects to look at SUBJECT_IDS = [1, 5, 6, 7, 8, 9, 11] #load camera parameters from the h36m dataset rcams = cameras2.load_cameras(FLAGS.cameras_path, SUBJECT_IDS) #loads 2d data from precomputed Stacked Hourglass detections train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.read_2d_predictions( actions, FLAGS.data_dir) #loads 3d poses, zero-centres and normalizes them train_set_3d, test_set_3d, data_mean_3d, data_std_3d, dim_to_ignore_3d, dim_to_use_3d, train_root_positions, test_root_positions = data_utils.read_3d_data( actions, FLAGS.data_dir, FLAGS.camera_frame, rcams, FLAGS.predict_14) device_count = {"GPU": 0} png_lib = [] #run a tensorflow inference session with tf.Session(config=tf.ConfigProto(device_count=device_count, allow_soft_placement=True)) as sess: #plt.figure(3) #load pre-trained model batch_size = 128 model = create_model(sess, actions, batch_size) #infinitely show 3d pose visualization while True: #wait for key to be pressed key = cv2.waitKey(1) & 0xFF _, frame = cv2.VideoCapture( 0).read() #ignore the other returned value #resize and rotate the incoming image frame frame, W, H = resize_img(frame) frame = cv2.rotate(frame, cv2.ROTATE_90_COUNTERCLOCKWISE) start = time.time() #run posenet inference on the frame joints_2d = estimate_pose(frame) #throw out confidence score and flatten _data = joints_2d[..., :2].flatten() #open pop-up and draw the keypoints found img2D = draw_2Dimg(frame, joints_2d, 1) #fake the thorax point by finding midpt between left and right shoulder lt_should_x = _data[10] lt_should_y = _data[11] rt_should_x = _data[12] rt_should_y = _data[13] thorax = midpoint(lt_should_x, lt_should_y, rt_should_x, rt_should_y) #print("testing thorax pt at ", thorax) #insert thorax into data where it should be, at index 1 _data = np.insert(_data, 2, thorax[0]) _data = np.insert(_data, 3, thorax[1]) #print("new _data is ", _data) _data = np.around(_data) #set xy to the array of 2d joint data xy = _data #create new 1x36 array of zeroes, which will store the 18 2d keypoints joints_array = np.zeros((1, 36)) joints_array[0] = [0 for i in range(36)] #index into our data array index = 0 #iterates 18 times for o in range(int(len(joints_array[0]) / 2)): #feed array with xy array (the 18 keypoints), but switch ordering: posenet to openpose for j in range(2): #print("o is", o, "j is", j, "index is ", index) index_into_posenet_data = order_pnet_to_openpose[o] * 2 + j #print("putting posenet[", index_into_posenet_data, "], value ", xy[index_into_posenet_data], " , into joints_array[0][", index, "]") joints_array[0][index] = xy[index_into_posenet_data] index += 1 #set _data to the array containing the 36 coordinates of the 2d keypts _data = joints_array[0] #print("_data is ", _data) #mapping all body parts for 3d-pose-baseline format (32 2d coordinates) for i in range(len(order)): #iterates 14 times #select which coordinateof this point: x or y for j in range(2): #create encoder input, switching around the order of the joint points enc_in[0][order[i] * 2 + j] = _data[i * 2 + j] #now enc_in contains 14 points (28 total coordinates) #at this pt enc_in should be array of 64 vals for j in range(2): #place hip at index 0 enc_in[0][0 * 2 + j] = (enc_in[0][1 * 2 + j] + enc_in[0][6 * 2 + j]) / 2 #place neck/nose at index 14 enc_in[0][14 * 2 + j] = (enc_in[0][15 * 2 + j] + enc_in[0][12 * 2 + j]) / 2 #place thorax at index 13 enc_in[0][13 * 2 + j] = 2 * enc_in[0][12 * 2 + j] - enc_in[0][14 * 2 + j] #set spine found by openpose spine_x = enc_in[0][24] spine_y = enc_in[0][25] #dim_to_use_2d is always [0 1 2 3 4 5 6 7 12 13 14 15 16 17 24 25 26 27 30 31 34 35 36 37 38 39 50 51 52 53 54 55] #take 32 entries of enc_in enc_in = enc_in[:, dim_to_use_2d] #find mean of 2d data mu = data_mean_2d[dim_to_use_2d] #find stdev of 2d data stddev = data_std_2d[dim_to_use_2d] #subtract mean and divide std for all enc_in = np.divide((enc_in - mu), stddev) #dropout keep probability dp = 1.0 #output tensor, initialize it to zeroes. We'll get 16 joints with 3d coordinates #this is list of numpy vectors that are the expected decoder outputs dec_out = np.zeros((1, 48)) dec_out[0] = [0 for i in range(48)] #get the 3d poses by running the 3d-pose-baseline inference. Model operates on 32 points _, _, poses3d = model.step(sess, enc_in, dec_out, dp, isTraining=False) #poses3d comes back as a 1x96 array (I guess its 32 points) end = time.time() #print("ELAPSED: ", end-start) #hold our 3d poses while we're doing some post-processing all_poses_3d = [] #un-normalize the input and output data using the means and stdevs enc_in = data_utils.unNormalizeData(enc_in, data_mean_2d, data_std_2d, dim_to_ignore_2d) poses3d = data_utils.unNormalizeData(poses3d, data_mean_3d, data_std_3d, dim_to_ignore_3d) #create a grid for drawing gs1 = gridspec.GridSpec(1, 1) #set spacing between axes gs1.update(wspace=-0.00, hspace=0.05) plt.axis('off') #fill all_poses_3d with the 3d poses predicted by the model step fxn all_poses_3d.append(poses3d) #vstack stacks arrays in sequence vertically (row wise) #this doesn't do anything in this case, as far as I can tell enc_in, poses3d = map(np.vstack, [enc_in, all_poses_3d]) subplot_idx, exidx = 1, 1 _max = 0 _min = 10000 #iterates once for i in range(poses3d.shape[0]): #iterate over all 32 points in poses3d for j in range(32): #save the last coordinate of this point into tmp tmp = poses3d[i][j * 3 + 2] #swap the second and third coordinates of this pt poses3d[i][j * 3 + 2] = poses3d[i][j * 3 + 1] poses3d[i][j * 3 + 1] = tmp #keep track of max of last coordinate if poses3d[i][j * 3 + 2] > _max: _max = poses3d[i][j * 3 + 2] if poses3d[i][j * 3 + 2] < _min: _min = poses3d[i][j * 3 + 2] #iterates once for i in range(poses3d.shape[0]): #iterate over all 32 points in poses3d (2nd and 3rd coords have all been swapped at this pt) for j in range(32): #change the third coord of this pt, subtracting it from sum of max and min third coord to get new value poses3d[i][j * 3 + 2] = _max - poses3d[i][j * 3 + 2] + _min #modify first coord of this pt by adding the x coord of the spine found by 2d model poses3d[i][j * 3] += (spine_x - 630) #modify third coord of this pt by adding 500 minus y coord of spine found by 2d model poses3d[i][j * 3 + 2] += (500 - spine_y) #Plot 3d predictions ax = plt.subplot(gs1[subplot_idx - 1], projection='3d') ax.view_init(18, -70) logger.debug(np.min(poses3d)) #TODO: if something happened with the data, reuse data from last frame (before_pose) p3d = poses3d #plot the 3d skeleton viz.show3Dpose(p3d, ax, lcolor="#9b59b6", rcolor="#2ecc71") #keep track of this poses3d in case we need to reuse it for next frame before_pose = poses3d #save this frame as a png in the ./png/ folder pngName = 'png/test_{0}.png'.format(str(framenum)) #print("pngName is ", pngName) plt.savefig(pngName) #plt.show() #read this frame which was just saved as png img = cv2.imread(pngName, 0) rect_cpy = img.copy() #show this frame cv2.imshow('3d-pose-baseline', rect_cpy) framenum += 1 #quit if q is pressed if key == ord('q'): break sess.close()
def main(_): # 出力用日付 now_str = "{0:%Y%m%d_%H%M%S}".format(datetime.datetime.now()) logger.debug("FLAGS.person_idx={0}".format(FLAGS.person_idx)) # ディレクトリ構成が変わったので、JSON出力と同階層に出力(2/9) if FLAGS.output is None: subdir = openpose_output_dir else: subdir = FLAGS.output os.makedirs(subdir, exist_ok=True) frame3d_dir = "{0}/frame3d".format(subdir) if os.path.exists(frame3d_dir): # 既にディレクトリがある場合、一旦削除 shutil.rmtree(frame3d_dir) os.makedirs(frame3d_dir) #関節位置情報ファイル posf = open(subdir + '/pos.txt', 'w') #正規化済みOpenpose位置情報ファイル smoothedf = open(subdir + '/smoothed.txt', 'w') #開始フレームインデックスファイル start_frame_f = open(subdir + '/start_frame.txt', 'w') idx = FLAGS.person_idx - 1 start_frame_index, smoothed = openpose_utils.read_openpose_json( "{0}/json".format(openpose_output_dir), idx, FLAGS.verbose == 3) # 開始フレームインデックスを保存 start_frame_f.write(str(start_frame_index)) start_frame_f.close() logger.info("reading and smoothing done. start feeding 3d-pose-baseline") logger.debug(smoothed) plt.figure(2) smooth_curves_plot = show_anim_curves(smoothed, plt) pngName = subdir + '/smooth_plot.png' smooth_curves_plot.savefig(pngName) enc_in = np.zeros((1, 64)) enc_in[0] = [0 for i in range(64)] actions = data_utils.define_actions(FLAGS.action) SUBJECT_IDS = [1, 5, 6, 7, 8, 9, 11] rcams = cameras.load_cameras(FLAGS.cameras_path, SUBJECT_IDS) train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.read_2d_predictions( actions, FLAGS.data_dir) train_set_3d, test_set_3d, data_mean_3d, data_std_3d, dim_to_ignore_3d, dim_to_use_3d, train_root_positions, test_root_positions = data_utils.read_3d_data( actions, FLAGS.data_dir, FLAGS.camera_frame, rcams, FLAGS.predict_14) # before_pose = None device_count = {"GPU": 1} png_lib = [] with tf.Session(config=tf.ConfigProto(device_count=device_count, allow_soft_placement=True)) as sess: #plt.figure(3) batch_size = 128 model = create_model(sess, actions, batch_size) # 入力画像のスケール調整のため、NeckからHipまでの距離を測定 length_neck2hip_mean = get_length_neck2hip_mean(smoothed) # 2D、3D結果の保存用リスト poses3d_list = [] poses2d_list = [] # 2dと3dのスケール比率計算のためのリスト length_2d_list = [] length_3d_list = [] for n, (frame, xy) in enumerate(smoothed.items()): if frame % 200 == 0: logger.info("calc idx {0}, frame {1}".format(idx, frame)) #if frame % 300 == 0: # print(frame) # map list into np array joints_array = np.zeros((1, 36)) joints_array[0] = [0 for i in range(36)] for o in range(len(joints_array[0])): #feed array with xy array joints_array[0][o] = xy[o] _data = joints_array[0] smoothedf.write(' '.join(map(str, _data))) smoothedf.write("\n") # mapping all body parts or 3d-pose-baseline format for i in range(len(order)): for j in range(2): # create encoder input enc_in[0][order[i] * 2 + j] = _data[i * 2 + j] for j in range(2): # Hip enc_in[0][0 * 2 + j] = (enc_in[0][1 * 2 + j] + enc_in[0][6 * 2 + j]) / 2 # Thorax # 3dPoseBaselineのThoraxの位置は、OpenPoseのNeckの位置より少し上のため調整する enc_in[0][13 * 2 + j] = 1.1 * enc_in[0][13 * 2 + j] - 0.1 * enc_in[0][0 * 2 + j] # Neck/Nose enc_in[0][14 * 2 + j] = (enc_in[0][15 * 2 + j] + enc_in[0][13 * 2 + j]) / 2 # Spine enc_in[0][12 * 2 + j] = (enc_in[0][0 * 2 + j] + enc_in[0][13 * 2 + j]) / 2 # set spine # spine_x = enc_in[0][24] # spine_y = enc_in[0][25] # logger.debug("enc_in - 1") # logger.debug(enc_in) poses2d = enc_in # 入力データの拡大 # neckからHipまでが110ピクセル程度になるように入力を拡大する # (教師データとスケールが大きく異なると精度が落ちるため) input_scaling_factor = 110 / length_neck2hip_mean enc_in = enc_in * input_scaling_factor enc_in = enc_in[:, dim_to_use_2d] mu = data_mean_2d[dim_to_use_2d] stddev = data_std_2d[dim_to_use_2d] enc_in = np.divide((enc_in - mu), stddev) dp = 1.0 dec_out = np.zeros((1, 48)) dec_out[0] = [0 for i in range(48)] _, _, poses3d = model.step(sess, enc_in, dec_out, dp, isTraining=False) all_poses_3d = [] enc_in = data_utils.unNormalizeData(enc_in, data_mean_2d, data_std_2d, dim_to_ignore_2d) poses3d = data_utils.unNormalizeData(poses3d, data_mean_3d, data_std_3d, dim_to_ignore_3d) gs1 = gridspec.GridSpec(1, 1) gs1.update(wspace=-0.00, hspace=0.05) # set the spacing between axes. plt.axis('off') all_poses_3d.append(poses3d) enc_in, poses3d = map(np.vstack, [enc_in, all_poses_3d]) subplot_idx, exidx = 1, 1 poses3d_list.append(poses3d[0]) poses2d_list.append(poses2d[0]) length_2d_list.append(sum_length_xy(poses2d[0], 2)) length_3d_list.append(sum_length_xy(poses3d[0], 3)) # OpenPose出力の(x, y)とBaseline出力のzから、3次元の位置を計算する # OpenPose出力値とBaseline出力値のスケール比率 # 骨格の長さの合計の比較することで、比率を推定 # 前後の91フレームで移動平均をとることで、結果を安定化する move_ave_length_2d = calc_move_average(length_2d_list, 91) move_ave_length_3d = calc_move_average(length_3d_list, 91) move_ave_length_2d[move_ave_length_2d == 0] = 1 # error防止 xy_scale = move_ave_length_3d / move_ave_length_2d # 以下の4つは仮の値で計算。多少違っていても、精度に影響はないと思う center_2d_x, center_2d_y = camera_center( openpose_output_dir) #動画の中心座標(動画の解像度の半分) logger.info("center_2d_x {0}".format(center_2d_x)) z_distance = 4000 # カメラから体までの距離(mm) 遠近の影響計算で使用 camera_incline = 0 # カメラの水平方向に対する下への傾き(度) teacher_camera_incline = 13 # 教師データ(Human3.6M)のカメラの傾き(下向きに平均13度) for frame, (poses3d, poses2d) in enumerate(zip(poses3d_list, poses2d_list)): # 誤差を減らすため、OpenPose出力の(x, y)と3dPoseBaseline出力のzから、3次元の位置を計算する poses3d_op_xy = np.zeros(96) for i in [0, 1, 2, 3, 6, 7, 8, 13, 15, 17, 18, 19, 25, 26, 27]: # Hipとの差分 dy = poses3d[i * 3 + 1] - poses3d[0 * 3 + 1] dz = poses3d[i * 3 + 2] - poses3d[0 * 3 + 2] # 教師データのカメラ傾きを補正 dz = dz - dy * math.tan( math.radians(teacher_camera_incline - camera_incline)) # 遠近によるx,yの拡大率 z_ratio = (z_distance + dz) / z_distance # x, yはOpenposeの値から計算 poses3d_op_xy[i * 3] = (poses2d[i * 2] - center_2d_x) * xy_scale[frame] * z_ratio poses3d_op_xy[i * 3 + 1] = (poses2d[i * 2 + 1] - center_2d_y) * xy_scale[frame] * z_ratio # zはBaselineの値から計算 poses3d_op_xy[i * 3 + 2] = dz # 12(Spine)、14(Neck/Nose)、15(Head)はOpenPoseの出力にないため、baseline(poses3d)から計算する for i in [12, 14, 15]: # 13(Thorax)は認識されることが多いため基準とする # 差分 dx = poses3d[i * 3] - poses3d[13 * 3] dy = poses3d[i * 3 + 1] - poses3d[13 * 3 + 1] dz = poses3d[i * 3 + 2] - poses3d[13 * 3 + 2] # 教師データのカメラ傾きを補正 dz = dz - dy * math.tan( math.radians(teacher_camera_incline - camera_incline)) # 13(Thorax)からの差分でx, y ,zを求める poses3d_op_xy[i * 3] = poses3d_op_xy[13 * 3] + dx poses3d_op_xy[i * 3 + 1] = poses3d_op_xy[13 * 3 + 1] + dy poses3d_op_xy[i * 3 + 2] = poses3d_op_xy[13 * 3 + 2] + dz # MMD上で少し顎を引くための処理 poses3d_op_xy[15 * 3] += 0.5 * (poses3d_op_xy[14 * 3] - poses3d_op_xy[13 * 3]) poses3d_op_xy[15 * 3 + 1] += 0.5 * (poses3d_op_xy[14 * 3 + 1] - poses3d_op_xy[13 * 3 + 1]) poses3d_op_xy[15 * 3 + 2] += 0.5 * (poses3d_op_xy[14 * 3 + 2] - poses3d_op_xy[13 * 3 + 2]) poses3d_list[frame] = poses3d_op_xy logger.info("calc ground y") # 最も高さが低い足の部位のYを取得(この座標系ではY値が大きい方が低い) foot_joint_no = [1, 2, 3, 6, 7, 8] max_pos = [] for frame, poses3d in enumerate(poses3d_list): max_pos.append(np.max([poses3d[i * 3 + 1] for i in foot_joint_no])) # 地面についている部位の位置(通常は足首)をY軸の0になるように移動する for frame, poses3d in enumerate(poses3d_list): # 120フレーム分の位置を取得 max_pos_slice = max_pos[int(np.max([0, frame - 60])):frame + 60] # 半分以上のフレームでは着地していると仮定し、メディアンを着地時の足の位置とする ankle_pos = np.median(max_pos_slice) poses3d_ground = np.zeros(96) for i in range(len(data_utils.H36M_NAMES)): poses3d_ground[i * 3] = poses3d[i * 3] poses3d_ground[i * 3 + 1] = poses3d[i * 3 + 1] - ankle_pos poses3d_ground[i * 3 + 2] = poses3d[i * 3 + 2] poses3d_list[frame] = poses3d_ground for frame, (poses3d, poses2d) in enumerate(zip(poses3d_list, poses2d_list)): if frame % 200 == 0: logger.info("output frame {}".format(frame)) # max = 0 # min = 10000 # logger.debug("enc_in - 2") # logger.debug(enc_in) for j in range(32): tmp = poses3d[j * 3 + 2] poses3d[j * 3 + 2] = -poses3d[j * 3 + 1] poses3d[j * 3 + 1] = tmp # if poses3d[i][j * 3 + 2] > max: # max = poses3d[i][j * 3 + 2] # if poses3d[i][j * 3 + 2] < min: # min = poses3d[i][j * 3 + 2] # for i in range(poses3d.shape[0]): # for j in range(32): # poses3d[i][j * 3 + 2] = max - poses3d[i][j * 3 + 2] + min # poses3d[i][j * 3] += (spine_x - 630) # poses3d[i][j * 3 + 2] += (500 - spine_y) # Plot 3d predictions ax = plt.subplot(gs1[subplot_idx - 1], projection='3d') ax.view_init(18, 280) # logger.debug(np.min(poses3d)) # if np.min(poses3d) < -1000 and before_pose is not None: # poses3d = before_pose p3d = poses3d # logger.debug("poses3d") # logger.debug(poses3d) if frame == 0: first_xyz = [0, 0, 0] first_xyz[0], first_xyz[1], first_xyz[2] = p3d[0], p3d[1], p3d[ 2] if level[FLAGS.verbose] <= logging.INFO: viz.show3Dpose(p3d, ax, lcolor="#9b59b6", rcolor="#2ecc71", add_labels=True, root_xyz=first_xyz) # 各フレームの単一視点からのはINFO時のみ pngName = frame3d_dir + '/tmp_{0:012d}.png'.format(frame) plt.savefig(pngName) png_lib.append(imageio.imread(pngName)) # before_pose = poses3d # 各フレームの角度別出力はデバッグ時のみ if level[FLAGS.verbose] == logging.DEBUG: for azim in [0, 45, 90, 135, 180, 225, 270, 315, 360]: ax2 = plt.subplot(gs1[subplot_idx - 1], projection='3d') ax2.view_init(18, azim) viz.show3Dpose(p3d, ax2, lcolor="#FF0000", rcolor="#0000FF", add_labels=True, root_xyz=first_xyz) pngName2 = frame3d_dir + '/tmp_{0:012d}_{1:03d}.png'.format( frame, azim) plt.savefig(pngName2) #関節位置情報の出力 write_pos_data(poses3d, ax, posf) posf.close() smoothedf.close() # INFO時は、アニメーションGIF生成 if level[FLAGS.verbose] <= logging.INFO: logger.info( "creating Gif {0}/movie_smoothing.gif, please Wait!".format( subdir)) imageio.mimsave('{0}/movie_smoothing.gif'.format(subdir), png_lib, fps=FLAGS.gif_fps) logger.info("Done!".format(pngName))
def main(_): done = [] enc_in = np.zeros((1, 64)) enc_in[0] = [0 for i in range(64)] actions = data_utils.define_actions(FLAGS.action) SUBJECT_IDS = [1, 5, 6, 7, 8, 9, 11] rcams = cameras.load_cameras(FLAGS.cameras_path, SUBJECT_IDS) train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.read_2d_predictions( actions, FLAGS.data_dir) train_set_3d, test_set_3d, data_mean_3d, data_std_3d, dim_to_ignore_3d, dim_to_use_3d, train_root_positions, test_root_positions = data_utils.read_3d_data( actions, FLAGS.data_dir, FLAGS.camera_frame, rcams, FLAGS.predict_14) device_count = {"GPU": 0} png_lib = [] with tf.Session(config=tf.ConfigProto(device_count=device_count, allow_soft_placement=True)) as sess: #plt.figure(3) batch_size = 128 model = create_model(sess, actions, batch_size) rows = 0 filename = "Realtimedata.xlsx" workbook = xlsxwriter.Workbook(filename) worksheet = workbook.add_worksheet() while True: key = cv2.waitKey(1) & 0xFF #logger.info("start reading data") # check for other file types list_of_files = glob.iglob("{0}/*".format( openpose_output_dir)) # You may use iglob in Python3 latest_file = "" try: latest_file = max(list_of_files, key=os.path.getctime) except ValueError: #empthy dir pass if not latest_file: continue try: _file = file_name = latest_file print(latest_file) if not os.path.isfile(_file): raise Exception("No file found!!, {0}".format(_file)) data = json.load(open(_file)) #take first person _data = data["people"][0]["pose_keypoints_2d"] xy = [] #ignore confidence score """for o in range(0,len(_data),3): xy.append(_data[o]) xy.append(_data[o+1])""" if len(_data) >= 53: #openpose incl. confidence score #ignore confidence score for o in range(0, len(_data), 3): xy.append(_data[o]) xy.append(_data[o + 1]) else: #tf-pose-estimation xy = _data frame_indx = re.findall("(\d+)", file_name) frame = int(frame_indx[0]) joints_array = np.zeros((1, 36)) joints_array[0] = [0 for i in range(36)] for o in range(len(joints_array[0])): #feed array with xy array joints_array[0][o] = xy[o] _data = joints_array[0] # mapping all body parts or 3d pose offline format for i in range(len(order)): for j in range(2): # create encoder input enc_in[0][order[i] * 2 + j] = _data[i * 2 + j] for j in range(2): # Hip enc_in[0][0 * 2 + j] = (enc_in[0][1 * 2 + j] + enc_in[0][6 * 2 + j]) / 2 # Neck/Nose enc_in[0][14 * 2 + j] = (enc_in[0][15 * 2 + j] + enc_in[0][12 * 2 + j]) / 2 # Thorax enc_in[0][ 13 * 2 + j] = 2 * enc_in[0][12 * 2 + j] - enc_in[0][14 * 2 + j] # set spine spine_x = enc_in[0][24] spine_y = enc_in[0][25] enc_in = enc_in[:, dim_to_use_2d] mu = data_mean_2d[dim_to_use_2d] stddev = data_std_2d[dim_to_use_2d] enc_in = np.divide((enc_in - mu), stddev) dp = 1.0 dec_out = np.zeros((1, 48)) dec_out[0] = [0 for i in range(48)] _, _, poses3d = model.step(sess, enc_in, dec_out, dp, isTraining=False) all_poses_3d = [] enc_in = data_utils.unNormalizeData(enc_in, data_mean_2d, data_std_2d, dim_to_ignore_2d) poses3d = data_utils.unNormalizeData(poses3d, data_mean_3d, data_std_3d, dim_to_ignore_3d) gs1 = gridspec.GridSpec(1, 1) gs1.update(wspace=-0.00, hspace=0.05) # set the spacing between axes. plt.axis('off') all_poses_3d.append(poses3d) enc_in, poses3d = map(np.vstack, [enc_in, all_poses_3d]) subplot_idx, exidx = 1, 1 _max = 0 _min = 10000 for i in range(poses3d.shape[0]): for j in range(32): tmp = poses3d[i][j * 3 + 2] poses3d[i][j * 3 + 2] = poses3d[i][j * 3 + 1] poses3d[i][j * 3 + 1] = tmp if poses3d[i][j * 3 + 2] > _max: _max = poses3d[i][j * 3 + 2] if poses3d[i][j * 3 + 2] < _min: _min = poses3d[i][j * 3 + 2] for i in range(poses3d.shape[0]): for j in range(32): poses3d[i][j * 3 + 2] = _max - poses3d[i][j * 3 + 2] + _min poses3d[i][j * 3] += (spine_x - 630) poses3d[i][j * 3 + 2] += (500 - spine_y) for val in min_vex: # f.write(str(val) + ' ' + str(p_vex[i]) + ''); # gait_list1.append({'IX': "%i" % val[0], # 'IY': "%i" % val[1], # 'Ix': "%i" % p_vex[i][0], # 'Iy': "%i" % p_vex[i][1], # 'Iz': "%i" % p_vex[i][2], # }) gait_list1.append(val[0]) gait_list1.append(val[1]) gait_list1.append(p_vex[i][0]) gait_list1.append(p_vex[i][1]) gait_list1.append(p_vex[i][2]) points.append( " %f %f %f %d %d %d 0\n" % (p_vex[i][0], p_vex[i][1], p_vex[i][2], 0, 255, 0)) x.append(p_vex[i][0]) y.append(p_vex[i][1]) z.append(p_vex[i][2]) i = i + 1 # Plot 3d predictions ax = plt.subplot(gs1[subplot_idx - 1], projection='3d') ax.view_init(18, -70) logger.debug(np.min(poses3d)) if np.min(poses3d) < -1000 and frame != 0: poses3d = before_pose p3d = poses3d '''gait_list1 = [] #enter file path below with open('key_joint_info.csv', 'w', newline='') as myfile: gait_list2.append(gait_list1) data1 = pd.DataFrame(gait_list2) wr = csv.writer(myfile, dialect = 'key_joint_info.csv' ) wr.writerow(p3d) wb.save(key_joint_info.csv)''' viz.show3Dpose(p3d, ax, lcolor="#9b59b6", rcolor="#2ecc71") col = 0 for i in p3d[0]: worksheet.write(rows, col, i) col += 1 #.append(i) rows += 1 before_pose = poses3d pngName = '{}_keypoints.png'.format(str(frame)) plt.savefig(pngName) #plt.show() img = cv2.imread(pngName, 0) rect_cpy = img.copy() cv2.imshow('3d-pose-realtime', rect_cpy) done.append(file_name) if key == ord('q'): break except Exception as e: print(e) sess.close()
def callback(data): if data.data == 'hello': print(data.data) else: global FLAGS global enc_in global actions global order global SUBJECT_IDS global rcams global train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d global train_set_3d, test_set_3d, data_mean_3d, data_std_3d, dim_to_ignore_3d, dim_to_use_3d, train_root_positions, test_root_positions global device_count input_data = np.array(eval(data.data)) print('input data shape:{0}'.format(input_data.shape)) tf.reset_default_graph() with tf.Session(config=tf.ConfigProto( device_count=device_count, allow_soft_placement=True)) as sess: batch_size = 128 model = create_model(sess, actions, batch_size) # map list into np array joints_array = np.zeros((1, 36)) joints_array[0] = [0 for i in range(36)] for o in range(len(joints_array[0])): #feed array with xy array joints_array[0] = input_data.flatten() _data = joints_array[0] # mapping all body parts or 3d-pose-baseline format for i in range(len(order)): for j in range(2): # create encoder input enc_in[0][order[i] * 2 + j] = _data[i * 2 + j] for j in range(2): # Hip enc_in[0][0 * 2 + j] = (enc_in[0][1 * 2 + j] + enc_in[0][6 * 2 + j]) / 2 # Neck/Nose enc_in[0][14 * 2 + j] = (enc_in[0][15 * 2 + j] + enc_in[0][12 * 2 + j]) / 2 # Thorax enc_in[0][13 * 2 + j] = 2 * enc_in[0][12 * 2 + j] - enc_in[0][14 * 2 + j] # set spine spine_x = enc_in[0][24] spine_y = enc_in[0][25] enc_in = enc_in[:, dim_to_use_2d] mu = data_mean_2d[dim_to_use_2d] stddev = data_std_2d[dim_to_use_2d] enc_in = np.divide((enc_in - mu), stddev) dp = 1.0 dec_out = np.zeros((1, 48)) dec_out[0] = [0 for i in range(48)] _, _, poses3d = model.step(sess, enc_in, dec_out, dp, isTraining=False) enc_in = data_utils.unNormalizeData(enc_in, data_mean_2d, data_std_2d, dim_to_ignore_2d) poses3d = data_utils.unNormalizeData(poses3d, data_mean_3d, data_std_3d, dim_to_ignore_3d) print('output data shape:{0}'.format(poses3d.shape))
def main(_): smoothed = read_openpose_json() logger.info("reading and smoothing done. start feeding 3d-pose-baseline") plt.figure(2) smooth_curves_plot = show_anim_curves(smoothed, plt) pngName = FLAGS.output_dirname + '/supplemental' + '/smooth_plot.png' smooth_curves_plot.savefig(pngName) if FLAGS.interpolation: framerange = len(smoothed.keys()) joint_rows = 36 array = np.concatenate(list(smoothed.values())) array_reshaped = np.reshape(array, (framerange, joint_rows)) multiplier = 0.1 multiplier_inv = 1 / multiplier out_array = np.array([]) for row in range(joint_rows): x = [] for frame in range(framerange): x.append(array_reshaped[frame, row]) frame = range(framerange) frame_resampled = np.arange(0, framerange, multiplier) spl = UnivariateSpline(frame, x, k=3) min_x, max_x = min(x), max(x) smooth_fac = max_x - min_x smooth_fac = smooth_fac * 125 spl.set_smoothing_factor(float(smooth_fac)) xnew = spl(frame_resampled) out_array = np.append(out_array, xnew) logger.info("done interpolating") a = np.array([]) for frame in range(int(framerange * multiplier_inv)): jnt_array = [] #print(frame) for jnt in range(joint_rows): jnt_array.append( out_array[jnt * int(framerange * multiplier_inv) + frame]) a = np.append(a, jnt_array) a = np.reshape(a, (int(framerange * multiplier_inv), joint_rows)) out_array = a interpolate_smoothed = {} for frame in range(int(framerange * multiplier_inv)): interpolate_smoothed[frame] = list(out_array[frame]) plt.figure(3) smoothed = interpolate_smoothed interpolate_curves_plot = show_anim_curves(smoothed, plt) pngName = FLAGS.output_dirname + '/supplemental' + '/interpolate_plot.png' interpolate_curves_plot.savefig(pngName) enc_in = np.zeros((1, 64)) enc_in[0] = [0 for i in range(64)] actions = data_utils.define_actions(FLAGS.action) SUBJECT_IDS = [1, 5, 6, 7, 8, 9, 11] rcams = cameras.load_cameras(FLAGS.cameras_path, SUBJECT_IDS) train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.read_2d_predictions( actions, FLAGS.data_dir) train_set_3d, test_set_3d, data_mean_3d, data_std_3d, dim_to_ignore_3d, dim_to_use_3d, train_root_positions, test_root_positions = data_utils.read_3d_data( actions, FLAGS.data_dir, FLAGS.camera_frame, rcams, FLAGS.predict_14) device_count = {"GPU": 1} png_lib = [] with tf.Session(config=tf.ConfigProto(device_count=device_count, allow_soft_placement=True)) as sess: plt.figure(4) batch_size = 128 model = create_model(sess, actions, batch_size) for n, (frame, xy) in enumerate(smoothed.items()): logger.info("calc frame {0}".format(frame)) # map list into np array joints_array = np.zeros((1, 36)) joints_array[0] = [0 for i in range(36)] for o in range(len(joints_array[0])): #feed array with xy array joints_array[0][o] = xy[o] _data = joints_array[0] # mapping all body parts or 3d-pose-baseline format for i in range(len(order)): for j in range(2): # create encoder input enc_in[0][order[i] * 2 + j] = _data[i * 2 + j] for j in range(2): # Hip enc_in[0][0 * 2 + j] = (enc_in[0][1 * 2 + j] + enc_in[0][6 * 2 + j]) / 2 # Neck/Nose enc_in[0][14 * 2 + j] = (enc_in[0][15 * 2 + j] + enc_in[0][12 * 2 + j]) / 2 # Thorax enc_in[0][13 * 2 + j] = 2 * enc_in[0][12 * 2 + j] - enc_in[0][14 * 2 + j] # set spine spine_x = enc_in[0][24] spine_y = enc_in[0][25] enc_in = enc_in[:, dim_to_use_2d] mu = data_mean_2d[dim_to_use_2d] stddev = data_std_2d[dim_to_use_2d] enc_in = np.divide((enc_in - mu), stddev) dp = 1.0 dec_out = np.zeros((1, 48)) dec_out[0] = [0 for i in range(48)] _, _, poses3d = model.step(sess, enc_in, dec_out, dp, isTraining=False) all_poses_3d = [] enc_in = data_utils.unNormalizeData(enc_in, data_mean_2d, data_std_2d, dim_to_ignore_2d) poses3d = data_utils.unNormalizeData(poses3d, data_mean_3d, data_std_3d, dim_to_ignore_3d) gs1 = gridspec.GridSpec(1, 1) gs1.update(wspace=-0.00, hspace=0.05) # set the spacing between axes. plt.axis('off') all_poses_3d.append(poses3d) enc_in, poses3d = map(np.vstack, [enc_in, all_poses_3d]) subplot_idx, exidx = 1, 1 _max = 0 _min = 10000 for i in range(poses3d.shape[0]): for j in range(32): tmp = poses3d[i][j * 3 + 2] poses3d[i][j * 3 + 2] = poses3d[i][j * 3 + 1] poses3d[i][j * 3 + 1] = tmp if poses3d[i][j * 3 + 2] > _max: _max = poses3d[i][j * 3 + 2] if poses3d[i][j * 3 + 2] < _min: _min = poses3d[i][j * 3 + 2] for i in range(poses3d.shape[0]): for j in range(32): poses3d[i][j * 3 + 2] = _max - poses3d[i][j * 3 + 2] + _min poses3d[i][j * 3] += (spine_x - 630) poses3d[i][j * 3 + 2] += (500 - spine_y) # Plot 3d predictions ax = plt.subplot(gs1[subplot_idx - 1], projection='3d') ax.view_init(18, -70) logger.debug(np.min(poses3d)) if np.min(poses3d) < -1000: poses3d = before_pose p3d = poses3d logger.debug(poses3d) viz.show3Dpose(p3d, ax, lcolor="#9b59b6", rcolor="#2ecc71") pngName = FLAGS.output_dirname + '/' + FLAGS.output_filename + '_{0}.png'.format( str(frame)) if FLAGS.write_output_img: plt.savefig(pngName) #png_lib.append(imageio.imread(pngName)) before_pose = poses3d if FLAGS.write_gif: logger.info("creating Gif movie_smoothing.gif, please Wait!") imageio.mimsave(FLAGS.output_dirname + '/supplemental' + '/movie_smoothing.gif', png_lib, fps=FLAGS.gif_fps) logger.info("Done!".format(pngName))
def main(_): smoothed = read_openpose_json() '''plt.figure(2) smooth_curves_plot = show_anim_curves(smoothed, plt) #return pngName = 'gif_output/smooth_plot.png' smooth_curves_plot.savefig(pngName) logger.info('writing gif_output/smooth_plot.png')''' if FLAGS.interpolation: logger.info("start interpolation") framerange = len(smoothed.keys()) joint_rows = 36 array = np.concatenate(list(smoothed.values())) array_reshaped = np.reshape(array, (framerange, joint_rows)) multiplier = FLAGS.multiplier multiplier_inv = 1 / multiplier out_array = np.array([]) for row in range(joint_rows): x = [] for frame in range(framerange): x.append(array_reshaped[frame, row]) frame = range(framerange) frame_resampled = np.arange(0, framerange, multiplier) spl = UnivariateSpline(frame, x, k=3) #relative smooth factor based on jnt anim curve min_x, max_x = min(x), max(x) smooth_fac = max_x - min_x smooth_resamp = 125 smooth_fac = smooth_fac * smooth_resamp spl.set_smoothing_factor(float(smooth_fac)) xnew = spl(frame_resampled) out_array = np.append(out_array, xnew) logger.info( "done interpolating. reshaping {0} frames, please wait!!".format( framerange)) a = np.array([]) for frame in range(int(framerange * multiplier_inv)): jnt_array = [] for jnt in range(joint_rows): jnt_array.append( out_array[jnt * int(framerange * multiplier_inv) + frame]) a = np.append(a, jnt_array) a = np.reshape(a, (int(framerange * multiplier_inv), joint_rows)) out_array = a interpolate_smoothed = {} for frame in range(int(framerange * multiplier_inv)): interpolate_smoothed[frame] = list(out_array[frame]) '''plt.figure(3) smoothed = interpolate_smoothed interpolate_curves_plot = show_anim_curves(smoothed, plt) pngName = 'gif_output/interpolate_{0}.png'.format(smooth_resamp) interpolate_curves_plot.savefig(pngName) logger.info('writing gif_output/interpolate_plot.png')''' enc_in = np.zeros((1, 64)) enc_in[0] = [0 for i in range(64)] actions = data_utils.define_actions(FLAGS.action) SUBJECT_IDS = [1, 5, 6, 7, 8, 9, 11] rcams = cameras.load_cameras(FLAGS.cameras_path, SUBJECT_IDS) train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.read_2d_predictions( actions, FLAGS.data_dir) train_set_3d, test_set_3d, data_mean_3d, data_std_3d, dim_to_ignore_3d, dim_to_use_3d, train_root_positions, test_root_positions = data_utils.read_3d_data( actions, FLAGS.data_dir, FLAGS.camera_frame, rcams, FLAGS.predict_14) device_count = {"GPU": 1} png_lib = [] with tf.Session(config=tf.ConfigProto(device_count=device_count, allow_soft_placement=True)) as sess: #plt.figure(3) batch_size = 128 model = create_model(sess, actions, batch_size) iter_range = len(smoothed.keys()) rows = 0 filename = "keyjoints_proband_222_01.xlsx" workbook = xlsxwriter.Workbook(filename) worksheet = workbook.add_worksheet() for n, (frame, xy) in enumerate(smoothed.items()): logger.info("calc frame {0}/{1}".format(frame, iter_range)) # map list into np array joints_array = np.zeros((1, 36)) joints_array[0] = [0 for i in range(36)] for o in range(len(joints_array[0])): #feed array with xy array joints_array[0][o] = xy[o] _data = joints_array[0] # mapping all body parts or 3d-pose-baseline format for i in range(len(order)): for j in range(2): # create encoder input enc_in[0][order[i] * 2 + j] = _data[i * 2 + j] for j in range(2): # Hip enc_in[0][0 * 2 + j] = (enc_in[0][1 * 2 + j] + enc_in[0][6 * 2 + j]) / 2 # Neck/Nose enc_in[0][14 * 2 + j] = (enc_in[0][15 * 2 + j] + enc_in[0][12 * 2 + j]) / 2 # Thorax enc_in[0][13 * 2 + j] = 2 * enc_in[0][12 * 2 + j] - enc_in[0][14 * 2 + j] # set spine spine_x = enc_in[0][24] spine_y = enc_in[0][25] enc_in = enc_in[:, dim_to_use_2d] mu = data_mean_2d[dim_to_use_2d] stddev = data_std_2d[dim_to_use_2d] enc_in = np.divide((enc_in - mu), stddev) dp = 1.0 dec_out = np.zeros((1, 48)) dec_out[0] = [0 for i in range(48)] _, _, poses3d = model.step(sess, enc_in, dec_out, dp, isTraining=False) all_poses_3d = [] enc_in = data_utils.unNormalizeData(enc_in, data_mean_2d, data_std_2d, dim_to_ignore_2d) poses3d = data_utils.unNormalizeData(poses3d, data_mean_3d, data_std_3d, dim_to_ignore_3d) gs1 = gridspec.GridSpec(1, 1) gs1.update(wspace=-0.00, hspace=0.05) # set the spacing between axes. plt.axis('off') all_poses_3d.append(poses3d) enc_in, poses3d = map(np.vstack, [enc_in, all_poses_3d]) subplot_idx, exidx = 1, 1 _max = 0 _min = 10000 for i in range(poses3d.shape[0]): for j in range(32): tmp = poses3d[i][j * 3 + 2] poses3d[i][j * 3 + 2] = poses3d[i][j * 3 + 1] poses3d[i][j * 3 + 1] = tmp if poses3d[i][j * 3 + 2] > _max: _max = poses3d[i][j * 3 + 2] if poses3d[i][j * 3 + 2] < _min: _min = poses3d[i][j * 3 + 2] for i in range(poses3d.shape[0]): for j in range(32): poses3d[i][j * 3 + 2] = _max - poses3d[i][j * 3 + 2] + _min poses3d[i][j * 3] += (spine_x - 630) poses3d[i][j * 3 + 2] += (500 - spine_y) # Plot 3d predictions #fig = plt.figure( figsize=(12.8, 7.2) ) ax = plt.subplot(gs1[subplot_idx - 1], projection='3d') #ax.view_init(18, -70) logger.debug(np.min(poses3d)) if np.min(poses3d) < -1000: poses3d = before_pose p3d = poses3d logger.debug(poses3d) #viz.Ax3DPose(ax, lcolor="#9b59b6", rcolor="#2ecc71") viz.show3Dpose(p3d, ax, lcolor="#9b59b6", rcolor="#2ecc71") #ax.scatter(p3d, c='b', marker='o') #result = viz.show3Dpose(p3d,ax) #print(p3d) ax.set_xlabel('X Label') ax.set_ylabel('Y Label') ax.set_zlabel('Z Label') ax.set_aspect('equal') #print(p3d) #plt.show() col = 0 for i in p3d[0]: worksheet.write(rows, col, i) col += 1 #.append(i) rows += 1 pngName = '{}_keypoints.png'.format(str(frame)) plt.savefig(pngName) if FLAGS.write_gif: png_lib.append(imageio.imread(pngName)) before_pose = poses3d workbook.close() if FLAGS.write_gif: if FLAGS.interpolation: #take every frame on gif_fps * multiplier_inv png_lib = np.array([ png_lib[png_image] for png_image in range(0, len(png_lib), int(multiplier_inv)) ]) logger.info("creating Gif gif_output/animation.gif, please Wait!") imageio.mimsave('gif_output/animation.gif', png_lib, fps=FLAGS.gif_fps) logger.info("Done!".format(pngName))
def main(_): actions_all = data_utils.define_actions( "All" ) # Load camera parameters SUBJECT_IDS = [1,5,6,7,8,9,11] rcams = cameras.load_cameras(FLAGS.cameras_path, SUBJECT_IDS) # Load 3d data and load (or create) 2d projections train_set_3d, test_set_3d, data_mean_3d, data_std_3d, dim_to_ignore_3d, dim_to_use_3d, train_root_positions, test_root_positions = data_utils.read_3d_data( actions_all, FLAGS.data_dir, FLAGS.camera_frame, rcams, FLAGS.predict_14 ) print("Finished Read 3D Data") # _, _, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.read_2d_predictions(actions_all, FLAGS.data_dir) _, _, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.create_2d_data( actions_all, FLAGS.data_dir, rcams ) print("Finished Read 2D Data") SH_TO_GT_PERM = np.array([SH_NAMES.index( h ) for h in H36M_NAMES if h != '' and h in SH_NAMES]) assert np.all( SH_TO_GT_PERM == np.array([6,2,1,0,3,4,5,7,8,9,13,14,15,12,11,10]) ) test_set = {} manipulation_dir = os.path.dirname(FLAGS.data_dir) manipulation_dir = os.path.dirname(manipulation_dir) manipulation_dir += '/manipulation_video/' manipulation_folders = glob.glob(manipulation_dir + '*') subj = 1 action = 'manipulation-video' for folder in manipulation_folders: seqname = os.path.basename( folder ) with h5py.File(folder + '/' + seqname +'.h5', 'r' ) as h5f: poses = h5f['poses'][:] # Permute the loaded data to make it compatible with H36M poses = poses[:,SH_TO_GT_PERM,:] # Reshape into n x (32*2) matrix poses = np.reshape(poses,[poses.shape[0], -1]) poses_final = np.zeros([poses.shape[0], len(H36M_NAMES)*2]) dim_to_use_x = np.where(np.array([x != '' and x != 'Neck/Nose' for x in H36M_NAMES]))[0] * 2 dim_to_use_y = dim_to_use_x+1 dim_to_use = np.zeros(len(SH_NAMES)*2,dtype=np.int32) dim_to_use[0::2] = dim_to_use_x dim_to_use[1::2] = dim_to_use_y poses_final[:,dim_to_use] = poses # If Uniframe Model Only keep the right 2d pose poses_final = poses_final[1::2,:] print(seqname, poses_final.shape) test_set[ (subj, action, seqname) ] = poses_final print("Finished Read Manipulations Videos") test_set_2d = data_utils.normalize_data( test_set, data_mean_2d, data_std_2d, dim_to_use_2d ) dim_to_use_12_manipulation_joints = np.array([3,4,5, 6,7,8, 9,10,11, 18,19,20, 21,22,23, 24,25,26, 51,52,53, 54,55,56, 57,58,59, 75,76,77, 78,79,80, 81,82,83]) print("Finished Normalize Manipualtion Videos") device_count = {"GPU": 0} if FLAGS.use_cpu else {"GPU": 1} with tf.Session(config=tf.ConfigProto( device_count = device_count )) as sess: # === Create the model === print("Creating %d layers of %d units." % (FLAGS.num_layers, FLAGS.linear_size)) batch_size = FLAGS.batch_size #Intial code is 64*2 model = predict_3dpose.create_model(sess, actions_all, batch_size) print("Model loaded") j = 0 for key2d in test_set_2d.keys(): (subj, b, fname) = key2d # if fname != specific_seqname + '.h5': # continue print( "Subject: {}, action: {}, fname: {}".format(subj, b, fname) ) enc_in = test_set_2d[ key2d ] n2d, _ = enc_in.shape # Split into about-same-size batches enc_in = np.array_split( enc_in, n2d // 1 ) all_poses_3d = [] for bidx in range( len(enc_in) ): # Dropout probability 0 (keep probability 1) for sampling dp = 1.0 anything = np.zeros((enc_in[bidx].shape[0], 48)) _, _, poses3d = model.step(sess, enc_in[bidx], anything, dp, isTraining=False) # Denormalize enc_in[bidx] = data_utils.unNormalizeData( enc_in[bidx], data_mean_2d, data_std_2d, dim_to_ignore_2d ) poses3d = data_utils.unNormalizeData( poses3d, data_mean_3d, data_std_3d, dim_to_ignore_3d ) all_poses_3d.append( poses3d ) # Put all the poses together enc_in, poses3d = map( np.vstack, [enc_in, all_poses_3d] ) enc_in, poses3d = map( np.vstack, [enc_in, poses3d] ) poses3d_12_manipulation = poses3d[:, dim_to_use_12_manipulation_joints] annotated_images = glob.glob(manipulation_dir + fname + '/info/*.xml') annotated_images = sorted(annotated_images) # 1080p = 1,920 x 1,080 fig = plt.figure(j, figsize=(10, 10)) gs1 = gridspec.GridSpec(3, 3) gs1.update(wspace=-0, hspace=0.1) # set the spacing between axes. plt.axis('off') subplot_idx = 1 nsamples = 3 for i in np.arange( nsamples ): # Plot 2d Detection ax1 = plt.subplot(gs1[subplot_idx-1]) img = mpimg.imread(manipulation_dir + fname + '/skeleton_cropped/' + os.path.basename(annotated_images[i]).split('_')[0] + '.jpg') ax1.imshow(img) # Plot 2d pose ax2 = plt.subplot(gs1[subplot_idx]) p2d = enc_in[i,:] viz.show2Dpose( p2d, ax2 ) ax2.invert_yaxis() # Plot 3d predictions # Compute first the procrustion and print error gt = getJ3dPosFromXML(annotated_images[i]) A = poses3d_12_manipulation[i, :].reshape(gt.shape) _, Z, T, b, c = procrustes.compute_similarity_transform(gt, A, compute_optimal_scale=True) sqerr = np.sqrt(np.sum((gt - (b*A.dot(T)) - c)**2, axis = 1)) print("{0} - {1} - Mean Error (mm) : {2}".format(fname, os.path.basename(annotated_images[i]), np.mean(sqerr))) ax3 = plt.subplot(gs1[subplot_idx+1], projection='3d') temp = poses3d[i,:].reshape((32, 3)) temp = c + temp.dot(T) #Do not scale p3d = temp.reshape((1, 96)) viz.show3Dpose( p3d, ax3, lcolor="#9b59b6", rcolor="#2ecc71" ) ax3.invert_zaxis() ax3.invert_yaxis() subplot_idx = subplot_idx + 3 plt.show() j += 1
def main(_): enc_in = np.zeros((1, 64)) enc_in[0] = [0 for i in range(64)] actions = data_utils.define_actions(FLAGS.action) SUBJECT_IDS = [1, 5, 6, 7, 8, 9, 11] rcams = cameras.load_cameras(FLAGS.cameras_path, SUBJECT_IDS) train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.read_2d_predictions( actions, FLAGS.data_dir) train_set_3d, test_set_3d, data_mean_3d, data_std_3d, dim_to_ignore_3d, dim_to_use_3d, train_root_positions, test_root_positions = data_utils.read_3d_data( actions, FLAGS.data_dir, FLAGS.camera_frame, rcams, FLAGS.predict_14) device_count = {"GPU": 1} png_lib = [] sess = tf.Session(config=tf.ConfigProto( device_count=device_count, allow_soft_placement=True)) # plt.figure(3) batch_size = 128 model = create_model(sess, actions, batch_size) '''''' openpose_dirs = sorted([ name for name in os.listdir(openpose_output_dir) if os.path.isdir(os.path.join(openpose_output_dir, name)) ]) openpose_dirs = [openpose_output_dir + file for file in openpose_dirs] error_dirs = [] for dir in openpose_dirs: # try: file_name = os.path.basename(dir) file_to_save = '../data/3d-pose-baseline/' + file_name + '.npz' if not os.path.isfile(file_to_save): # If haven't already processed smoothed = read_openpose_json(dir) plt.figure(2) # smooth_curves_plot = show_anim_curves(smoothed, plt) # return # pngName = 'gif_output/smooth_plot.png' # smooth_curves_plot.savefig(pngName) # logger.info('writing gif_output/smooth_plot.png') if FLAGS.interpolation: logger.info("start interpolation") framerange = len(smoothed.keys()) joint_rows = 36 array = np.concatenate(list(smoothed.values())) array_reshaped = np.reshape(array, (framerange, joint_rows)) multiplier = FLAGS.multiplier multiplier_inv = 1 / multiplier out_array = np.array([]) for row in range(joint_rows): x = [] for frame in range(framerange): x.append(array_reshaped[frame, row]) frame = range(framerange) frame_resampled = np.arange(0, framerange, multiplier) spl = UnivariateSpline(frame, x, k=3) # relative smooth factor based on jnt anim curve min_x, max_x = min(x), max(x) smooth_fac = max_x - min_x smooth_resamp = 125 smooth_fac = smooth_fac * smooth_resamp spl.set_smoothing_factor(float(smooth_fac)) xnew = spl(frame_resampled) out_array = np.append(out_array, xnew) logger.info("done interpolating. reshaping {0} frames, please wait!!".format(framerange)) a = np.array([]) for frame in range(int(framerange * multiplier_inv)): jnt_array = [] for jnt in range(joint_rows): jnt_array.append(out_array[jnt * int(framerange * multiplier_inv) + frame]) a = np.append(a, jnt_array) a = np.reshape(a, (int(framerange * multiplier_inv), joint_rows)) out_array = a interpolate_smoothed = {} for frame in range(int(framerange * multiplier_inv)): interpolate_smoothed[frame] = list(out_array[frame]) # plt.figure(3) smoothed = interpolate_smoothed # interpolate_curves_plot = show_anim_curves(smoothed, plt) # pngName = 'gif_output/interpolate_{0}.png'.format(smooth_resamp) # interpolate_curves_plot.savefig(pngName) # logger.info('writing gif_output/interpolate_plot.png') iter_range = len(smoothed.keys()) positions_3d = [] for n, (frame, xy) in enumerate(smoothed.items()): logger.info("calc frame {0}/{1}".format(frame, iter_range)) # map list into np array joints_array = np.zeros((1, 36)) joints_array[0] = [0 for i in range(36)] for o in range(len(joints_array[0])): #feed array with xy array joints_array[0][o] = xy[o] _data = joints_array[0] # mapping all body parts or 3d-pose-baseline format for i in range(len(order)): for j in range(2): # create encoder input enc_in[0][order[i] * 2 + j] = _data[i * 2 + j] for j in range(2): # Hip enc_in[0][0 * 2 + j] = (enc_in[0][1 * 2 + j] + enc_in[0][6 * 2 + j]) / 2 # Neck/Nose enc_in[0][14 * 2 + j] = (enc_in[0][15 * 2 + j] + enc_in[0][12 * 2 + j]) / 2 # Thorax enc_in[0][13 * 2 + j] = 2 * enc_in[0][12 * 2 + j] - enc_in[0][14 * 2 + j] # set spine spine_x = enc_in[0][24] spine_y = enc_in[0][25] enc_in = enc_in[:, dim_to_use_2d] mu = data_mean_2d[dim_to_use_2d] stddev = data_std_2d[dim_to_use_2d] enc_in = np.divide((enc_in - mu), stddev) dp = 1.0 dec_out = np.zeros((1, 48)) dec_out[0] = [0 for i in range(48)] _, _, poses3d = model.step(sess, enc_in, dec_out, dp, isTraining=False) all_poses_3d = [] enc_in = data_utils.unNormalizeData(enc_in, data_mean_2d, data_std_2d, dim_to_ignore_2d) poses3d = data_utils.unNormalizeData(poses3d, data_mean_3d, data_std_3d, dim_to_ignore_3d) gs1 = gridspec.GridSpec(1, 1) gs1.update(wspace=-0.00, hspace=0.05) # set the spacing between axes. plt.axis('off') all_poses_3d.append( poses3d ) enc_in, poses3d = map( np.vstack, [enc_in, all_poses_3d] ) subplot_idx, exidx = 1, 1 _max = 0 _min = 10000 for i in range(poses3d.shape[0]): for j in range(32): tmp = poses3d[i][j * 3 + 2] poses3d[i][j * 3 + 2] = poses3d[i][j * 3 + 1] poses3d[i][j * 3 + 1] = tmp if poses3d[i][j * 3 + 2] > _max: _max = poses3d[i][j * 3 + 2] if poses3d[i][j * 3 + 2] < _min: _min = poses3d[i][j * 3 + 2] for i in range(poses3d.shape[0]): for j in range(32): poses3d[i][j * 3 + 2] = _max - poses3d[i][j * 3 + 2] + _min poses3d[i][j * 3] += (spine_x - 630) poses3d[i][j * 3 + 2] += (500 - spine_y) # Plot 3d predictions ax = plt.subplot(gs1[subplot_idx - 1], projection='3d') ax.view_init(18, -70) logger.debug(np.min(poses3d)) if np.min(poses3d) < -1000 and frame != 0: poses3d = before_pose p3d = poses3d logger.debug(poses3d) viz.show3Dpose(p3d, ax, lcolor="#9b59b6", rcolor="#2ecc71") positions_3d.append(p3d) pngName = 'png/pose_frame_{0}.png'.format(str(frame).zfill(12)) plt.savefig(pngName) if FLAGS.write_gif: png_lib.append(imageio.imread(pngName)) before_pose = poses3d np.savez_compressed(file_to_save, positions_3d=positions_3d) if FLAGS.write_gif: if FLAGS.interpolation: #take every frame on gif_fps * multiplier_inv png_lib = np.array([png_lib[png_image] for png_image in range(0,len(png_lib), int(multiplier_inv)) ]) logger.info("creating Gif png/animation_" + file_name + ".gif, please Wait!") imageio.mimsave("gif_output/animation_" + file_name + ".gif", png_lib, fps=FLAGS.gif_fps) logger.info("Done!".format(pngName)) # except: # error_dirs.append(dir) with open('error_op_3d_dirs.txt', 'w') as f: for item in error_dirs: f.write("%s\n" % item)