def perform_noisy_actions(train_set_noisy, train_set_2d_org, camera_frame, rcams, predict_14, percent): #validate(train_set, train_set) print('Performing noisy actions now..') train_set_3d_noisy = addNoiseTo3D(train_set_noisy, percent) #got the 3d data and now do 2d things train_set_2d_noisy = du.project_to_cameras(train_set_3d_noisy, rcams) #there can be augmented 2d data train_set_2d_org.update(train_set_2d_noisy) # Compute normalization statistics. complete_train = copy.deepcopy(np.vstack(train_set_2d_org.values())) data_mean, data_std, dim_to_ignore, dim_to_use = du.normalization_stats( complete_train, dim=2) # Divide every dimension independently train_set_2d_org = du.normalize_data(train_set_2d_org, data_mean, data_std, dim_to_use) print("For noisy, mean :") print(data_mean) print("For noisy, std :") print(data_std) print('noisy actions done..') return train_set_2d_org, data_mean, data_std
def read_all_data(self, actions, data_dir, one_hot=False): """ Loads data for training/testing and normalizes it. Args actions: list of strings (actions) to load seq_length_in: number of frames to use in the burn-in sequence seq_length_out: number of frames to use in the output sequence data_dir: directory to load the data from one_hot: whether to use one-hot encoding per action Returns train_set: dictionary with normalized training data test_set: dictionary with test data data_mean: d-long vector with the mean of the training data data_std: d-long vector with the standard dev of the training data dim_to_ignore: dimensions that are not used becaused stdev is too small dim_to_use: dimensions that we are actually using in the model """ train_subject_ids = [1, 6, 7, 8, 9, 11] test_subject_ids = [5] train_set, complete_train = data_utils.load_data( data_dir, train_subject_ids, actions, one_hot) test_set, complete_test = data_utils.load_data(data_dir, test_subject_ids, actions, one_hot) # Compute normalization stats data_mean, data_std, dim_to_ignore, dim_to_use = data_utils.normalization_stats( complete_train) # Normalize -- subtract mean, divide by stdev train_set = data_utils.normalize_data(train_set, data_mean, data_std, dim_to_use, actions, one_hot) test_set = data_utils.normalize_data(test_set, data_mean, data_std, dim_to_use, actions, one_hot) print("done reading data.") self.train_set = train_set self.test_set = test_set self.data_mean = data_mean self.data_std = data_std self.dim_to_ignore = dim_to_ignore self.dim_to_use = dim_to_use self.train_keys = list(self.train_set.keys())
def read_all_data(actions=walking_lst, seq_length_in=50, seq_length_out=25, data_dir="./data/h3.6m/dataset", one_hot=True): """ Loads data for training/testing and normalizes it. Args actions: list of strings (actions) to load seq_length_in: number of frames to use in the burn-in sequence seq_length_out: number of frames to use in the output sequence data_dir: directory to load the data from one_hot: whether to use one-hot encoding per action Returns train_set: dictionary with normalized training data test_set: dictionary with test data data_mean: d-long vector with the mean of the training data data_std: d-long vector with the standard dev of the training data dim_to_ignore: dimensions that are not used becaused stdev is too small dim_to_use: dimensions that we are actually using in the model """ # === Read training data === print("Reading training data (seq_len_in: {0}, seq_len_out {1}).".format( seq_length_in, seq_length_out)) train_subject_ids = [1, 6, 7, 8, 9, 11] test_subject_ids = [5] train_set, complete_train = data_utils.load_data(data_dir, train_subject_ids, actions, one_hot) test_set, complete_test = data_utils.load_data(data_dir, test_subject_ids, actions, one_hot) # Compute normalization stats data_mean, data_std, dim_to_ignore, dim_to_use = data_utils.normalization_stats( complete_train) # Normalize -- subtract mean, divide by stdev train_set = data_utils.normalize_data(train_set, data_mean, data_std, dim_to_use, actions, one_hot) test_set = data_utils.normalize_data(test_set, data_mean, data_std, dim_to_use, actions, one_hot) print("done reading data.") return train_set, test_set, data_mean, data_std, dim_to_ignore, dim_to_use
poses = h5f['poses'][:] print(poses.shape) poses = poses[:, SH_TO_GT_PERM, :] #按照H36M的顺序来排列stacked hourglass的数据,这一步之后就已经是h3.6m的坐标系 poses = np.reshape(poses, [poses.shape[0], -1]) print(poses.shape) poses_final = np.zeros([poses.shape[0], len(H36M_NAMES) * 2]) #final的size是1612*64 #*2是因为每个坐标的xy都放在一起即(x1,y1,x2,y2...),且数据是二维的 dim_to_use_x = np.where( np.array([x != '' and x != 'Neck/Nose' for x in H36M_NAMES]))[0] * 2 print(dim_to_use_x) dim_to_use_y = dim_to_use_x + 1 dim_to_use = np.zeros(len(SH_NAMES) * 2, dtype=np.int32) #size为32 dim_to_use[0::2] = dim_to_use_x print(dim_to_use) dim_to_use[1::2] = dim_to_use_y print(dim_to_use) poses_final[:, dim_to_use] = poses #将pose填入poses_final当中 print(poses_final.shape) data_mean, data_std, dim_to_ignore, dim_to_use = data_utils.normalization_stats( poses_final, dim=2) print(data_mean.shape, data_std.shape, dim_to_ignore, dim_to_use) data = poses_final[:, dim_to_use] mu = data_mean[dim_to_use] stddev = data_std[dim_to_use] data_out = np.divide((data - mu), stddev) print(data_out.shape)
def read_all_data(actions, data_dir, one_hot, GT=False): """ Loads data for training/testing and normalizes it. Args Returns """ # === Read training data === print("Reading training data") ##### #need to add one-hot vector to each frame later ##### for i, action in enumerate(actions): action_seq = np.load(data_dir + '/' + action + '.npz')['seq'] action_seq_len = np.load(data_dir + '/' + action + '.npz')['seq_len'] all_seq = np.concatenate( (all_seq, action_seq), axis=0) if i > 0 else action_seq all_seq_len = np.concatenate((all_seq_len, action_seq_len), axis=0) \ if i > 0 else action_seq_len for i in range(len(all_seq)): if all_seq[i, 0, 0, 0, 0] > all_seq[i, 1, 0, 0, 0]: all_seq[i, :, :, :, 1] = -all_seq[i, :, :, :, 1] """ all_seq: [data_index, skeleton_num, frame_len, joints, xyz] """ all_seq = data_utils.rotate_data(all_seq) shape = all_seq.shape all_seq = np.reshape(all_seq, [shape[0], shape[1], shape[2], -1]) np.savez('val.npz', seq=all_seq) from sklearn.model_selection import train_test_split train_seq, test_seq, train_seq_len, test_seq_len = train_test_split( all_seq, all_seq_len, test_size=0.25, random_state=1) if GT: return test_seq for i, (seq, seq_len) in enumerate(zip(train_seq, train_seq_len)): try: complete_seq = np.concatenate((complete_seq, seq[: ,:int(seq_len)]), axis=1) if i > 0 \ else seq[:, :int(seq_len)] except TypeError: import pdb pdb.set_trace() complete_real_person, complete_character = complete_seq ''' #for data_visualization np.savez('all_data_mirror.npz', real_person=complete_real_person, character=complete_character, ground_truth=complete_character, loss=0) sys.exit() ''' # Compute normalization stats rp_stats = data_utils.normalization_stats(complete_real_person, FLAGS.dim_to_compressed) ch_stats = data_utils.normalization_stats(complete_character, FLAGS.dim_to_compressed) # Normalize -- subtract mean, divide by stdev train_shape = train_seq.shape test_shape = test_seq.shape if rp_stats[4] is not None: normalized_train_seq = np.zeros([ train_shape[0], train_shape[1], train_shape[2], rp_stats[4].get_params()['n_components'] ]) normalized_test_seq = np.zeros([ test_shape[0], test_shape[1], test_shape[2], rp_stats[4].get_params()['n_components'] ]) else: normalized_train_seq = np.zeros(train_shape) normalized_test_seq = np.zeros(test_shape) normalized_train_seq[:, 0] = data_utils.normalize_data( train_seq[:, 0], rp_stats, actions, one_hot) normalized_train_seq[:, 1] = data_utils.normalize_data( train_seq[:, 1], ch_stats, actions, one_hot) normalized_test_seq[:, 0] = data_utils.normalize_data(test_seq[:, 0], rp_stats, actions, one_hot) normalized_test_seq[:, 1] = data_utils.normalize_data(test_seq[:, 1], ch_stats, actions, one_hot) print("done reading data.") return normalized_train_seq, normalized_test_seq, train_seq_len, test_seq_len, rp_stats, ch_stats, \ max(all_seq_len)
import data_utils from viz import Ax3DPose data_dir = './data/h36m/dataset' train_subject_ids = [1, 6, 7, 8, 9, 11] test_subject_ids = [5] actions = [ "walking", "eating", "smoking", "discussion", "directions", "greeting", "phoning", "posing", "purchases", "sitting", "sittingdown", "takingphoto", "waiting", "walkingdog", "walkingtogether" ] one_hot = True train_set, complete_train = data_utils.load_data(data_dir, train_subject_ids, actions, one_hot) test_set, complete_test = data_utils.load_data(data_dir, test_subject_ids, actions, one_hot) data_mean, data_std, dim_to_ignore, dim_to_use = data_utils.normalization_stats( complete_train) train_set = data_utils.normalize_data(train_set, data_mean, data_std, dim_to_use, actions, one_hot) test_set = data_utils.normalize_data(test_set, data_mean, data_std, dim_to_use, actions, one_hot) print("done reading data.") print(train_set[(1, "walking", 1, "even")])
def __init__(self, sampling=True): # def create_model(actions, sampling=False): # Learning self.seq_out = 25 parser = argparse.ArgumentParser( description='Train RNN for human pose estimation') parser.add_argument('--learning_rate', dest='learning_rate', help='Learning rate', default=0.005, type=float) parser.add_argument( '--learning_rate_decay_factor', dest='learning_rate_decay_factor', help='Learning rate is multiplied by this much. 1 means no decay.', default=0.95, type=float) parser.add_argument('--learning_rate_step', dest='learning_rate_step', help='Every this many steps, do decay.', default=10000, type=int) parser.add_argument('--batch_size', dest='batch_size', help='Batch size to use during training.', default=16, type=int) parser.add_argument('--max_gradient_norm', dest='max_gradient_norm', help='Clip gradients to this norm.', default=5, type=float) parser.add_argument('--iterations', dest='iterations', help='Iterations to train for.', default=1e5, type=int) parser.add_argument('--test_every', dest='test_every', help='', default=100, type=int) # Architecture parser.add_argument('--architecture', dest='architecture', help='Seq2seq architecture to use: [basic, tied].', default='tied', type=str) parser.add_argument( '--loss_to_use', dest='loss_to_use', help='The type of loss to use, supervised or sampling_based', default='sampling_based', type=str) parser.add_argument( '--residual_velocities', dest='residual_velocities', help='Add a residual connection that effectively models velocities', action='store_true', default=False) parser.add_argument('--size', dest='size', help='Size of each model layer.', default=1024, type=int) parser.add_argument('--num_layers', dest='num_layers', help='Number of layers in the model.', default=1, type=int) parser.add_argument( '--seq_length_in', dest='seq_length_in', help='Number of frames to feed into the encoder. 25 fp', default=50, type=int) parser.add_argument( '--seq_length_out', dest='seq_length_out', help='Number of frames that the decoder has to predict. 25fps', default=self.seq_out, type=int) parser.add_argument('--omit_one_hot', dest='omit_one_hot', help='', action='store_true', default=False) # Directories parser.add_argument( '--data_dir', dest='data_dir', help='Data directory', default=os.path.normpath("./data_IK/h3.6m/dataset"), type=str) parser.add_argument('--train_dir', dest='train_dir', help='Training directory', default=os.path.normpath("./experiments/"), type=str) parser.add_argument( '--action', dest='action', help= 'The action to train on. all means all the actions, all_periodic means walking, eating and smoking', default="posing", type=str) parser.add_argument('--use_cpu', dest='use_cpu', help='', action='store_true', default=False) # parser.add_argument('--load', dest='load', # help='Try to load a previous checkpoint.', # default=, type=int) parser.add_argument('--sample', dest='sample', help='Set to True for sampling.', action='store_true', default=False) self.args = parser.parse_args() self.model = seq2seq_model.Seq2SeqModel( self.args.architecture, self.args.seq_length_in if not sampling else 50, self.args.seq_length_out if not sampling else self.seq_out, self.args.size, # hidden layer size self.args.num_layers, self.args.max_gradient_norm, self.args.batch_size, self.args.learning_rate, self.args.learning_rate_decay_factor, self.args.loss_to_use if not sampling else "sampling_based", 1, not self.args.omit_one_hot, self.args.residual_velocities, dtype=torch.float32) print("Loading model") self.model = torch.load('./25_best_model_7100') self.model.source_seq_len = 50 self.model.target_seq_len = self.seq_out act = ["posing"] train_subject_ids = [1, 6] train_set, complete_train = data_utils.load_data( self.args.data_dir, train_subject_ids, act, not self.args.omit_one_hot) self.data_mean, self.data_std, self.dim_to_ignore, self.dim_to_use = data_utils.normalization_stats( complete_train)
def main(_): smoothed = read_openpose_json_mydata() plt.figure(2) smooth_curves_plot = show_anim_curves(smoothed, plt) # return pngName = 'gif_output/smooth_plot.png' smooth_curves_plot.savefig(pngName) logger.info('writing gif_output/smooth_plot.png') data_dir = '/home/lyunfan/NYU_summer_intern/3d_pose_baseline_pytorch/data/' stat_3d = torch.load(os.path.join(data_dir, 'stat_3d.pth.tar')) train_2d = torch.load(os.path.join(data_dir, 'train_2d_ft.pth.tar')) complete_train = copy.deepcopy(np.vstack(train_2d.values())) data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.normalization_stats( complete_train, dim=2) data_mean_3d, data_std_3d, dim_to_ignore_3d, dim_to_use_3d = stat_3d['mean'], stat_3d['std'], stat_3d['dim_ignore'], \ stat_3d['dim_use'], if FLAGS.interpolation: logger.info("start interpolation") framerange = len(smoothed.keys()) joint_rows = 36 array = np.concatenate(list(smoothed.values())) array_reshaped = np.reshape(array, (framerange, joint_rows)) multiplier = FLAGS.multiplier multiplier_inv = 1 / multiplier out_array = np.array([]) for row in range(joint_rows): x = [] for frame in range(framerange): x.append(array_reshaped[frame, row]) frame = range(framerange) frame_resampled = np.arange(0, framerange, multiplier) spl = UnivariateSpline(frame, x, k=3) # relative smooth factor based on jnt anim curve min_x, max_x = min(x), max(x) smooth_fac = max_x - min_x smooth_resamp = 125 smooth_fac = smooth_fac * smooth_resamp spl.set_smoothing_factor(float(smooth_fac)) xnew = spl(frame_resampled) out_array = np.append(out_array, xnew) logger.info( "done interpolating. reshaping {0} frames, please wait!!".format( framerange)) a = np.array([]) for frame in range(int(framerange * multiplier_inv)): jnt_array = [] for jnt in range(joint_rows): jnt_array.append( out_array[jnt * int(framerange * multiplier_inv) + frame]) a = np.append(a, jnt_array) a = np.reshape(a, (int(framerange * multiplier_inv), joint_rows)) out_array = a interpolate_smoothed = {} for frame in range(int(framerange * multiplier_inv)): interpolate_smoothed[frame] = list(out_array[frame]) plt.figure(3) smoothed = interpolate_smoothed interpolate_curves_plot = show_anim_curves(smoothed, plt) pngName = 'gif_output/interpolate_{0}.png'.format(smooth_resamp) interpolate_curves_plot.savefig(pngName) logger.info('writing gif_output/interpolate_plot.png') enc_in = np.zeros((1, 64)) enc_in[0] = [0 for i in range(64)] actions = data_utils.define_actions(FLAGS.action) # SUBJECT_IDS = [1, 5, 6, 7, 8, 9, 11] # rcams = cameras.load_cameras(FLAGS.cameras_path, SUBJECT_IDS) # train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.read_2d_predictions( # actions, FLAGS.data_dir) # train_set_3d, test_set_3d, data_mean_3d, data_std_3d, dim_to_ignore_3d, dim_to_use_3d, train_root_positions, test_root_positions = data_utils.read_3d_data( # actions, FLAGS.data_dir, FLAGS.camera_frame, rcams, FLAGS.predict_14) device_count = {"GPU": 1} png_lib = [] before_pose = None with tf.Session(config=tf.ConfigProto(device_count=device_count, allow_soft_placement=True)) as sess: # plt.figure(3) batch_size = 128 model = create_model(sess, actions, batch_size) iter_range = len(smoothed.keys()) export_units = {} twod_export_units = {} for n, (frame, xy) in enumerate(smoothed.items()): logger.info("calc frame {0}/{1}".format(frame, iter_range)) # map list into np array # joints_array = np.zeros((1, 36)) # joints_array[0] = [0 for i in range(36)] # for o in range(len(joints_array[0])): # # feed array with xy array # joints_array[0][o] = float(xy[o]) # _data = joints_array[0] # NEWLY ADDED enc_in = np.zeros((1, 64)) enc_in[0] = [0 for i in range(64)] twod_export_units[frame] = {} for abs_b, __n in enumerate(range(0, len(xy), 2)): twod_export_units[frame][abs_b] = { "translate": [xy[__n], xy[__n + 1]] } _data = [float(i) for i in xy][:36] # mapping all body parts or 3d-pose-baseline format for i in range(len(order)): for j in range(2): # create encoder input enc_in[0][order[i] * 2 + j] = _data[i * 2 + j] for j in range(2): # Hip enc_in[0][0 * 2 + j] = (enc_in[0][1 * 2 + j] + enc_in[0][6 * 2 + j]) / 2 # Neck/Nose enc_in[0][14 * 2 + j] = (enc_in[0][15 * 2 + j] + enc_in[0][12 * 2 + j]) / 2 # Thorax enc_in[0][13 * 2 + j] = 2 * enc_in[0][12 * 2 + j] - enc_in[0][14 * 2 + j] # set spine spine_x = enc_in[0][24] spine_y = enc_in[0][25] enc_in = enc_in[:, dim_to_use_2d] # mu = data_mean_2d[dim_to_use_2d] mu = data_mean_2d # stddev = data_std_2d[dim_to_use_2d] stddev = data_std_2d enc_in = np.divide((enc_in - mu), stddev) dp = 1.0 dec_out = np.zeros((1, 48)) dec_out[0] = [0 for i in range(48)] _, _, poses3d = model.step(sess, enc_in, dec_out, dp, isTraining=False) all_poses_3d = [] enc_in = data_utils.unNormalizeData2d(enc_in, data_mean_2d, data_std_2d, dim_to_use_2d) poses3d = data_utils.unNormalizeData(poses3d, data_mean_3d, data_std_3d, dim_to_ignore_3d) gs1 = gridspec.GridSpec(1, 1) gs1.update(wspace=-0.00, hspace=0.05) # set the spacing between axes. plt.axis('off') all_poses_3d.append(poses3d) enc_in, poses3d = map(np.vstack, [enc_in, all_poses_3d]) subplot_idx, exidx = 1, 1 _max = 0 _min = 10000 for i in range(poses3d.shape[0]): for j in range(32): tmp = poses3d[i][j * 3 + 2] poses3d[i][j * 3 + 2] = poses3d[i][j * 3 + 1] poses3d[i][j * 3 + 1] = tmp if poses3d[i][j * 3 + 2] > _max: _max = poses3d[i][j * 3 + 2] if poses3d[i][j * 3 + 2] < _min: _min = poses3d[i][j * 3 + 2] for i in range(poses3d.shape[0]): for j in range(32): poses3d[i][j * 3 + 2] = _max - poses3d[i][j * 3 + 2] + _min poses3d[i][j * 3] += (spine_x - 630) poses3d[i][j * 3 + 2] += (500 - spine_y) # Plot 3d predictions ax = plt.subplot(gs1[subplot_idx - 1], projection='3d') ax.view_init(18, -70) # if FLAGS.cache_on_fail: # if np.min(poses3d) < -1000 and before_pose: # poses3d = before_pose p3d = poses3d logger.info("frame score {0}".format(np.min(poses3d))) x, y, z = [[] for _ in range(3)] if not poses3d is None: to_export = poses3d.tolist()[0] else: to_export = [0.0 for _ in range(96)] logger.info("export {0}".format(to_export)) for o in range(0, len(to_export), 3): x.append(to_export[o]) y.append(to_export[o + 1]) z.append(to_export[o + 2]) export_units[frame] = {} for jnt_index, (_x, _y, _z) in enumerate(zip(x, y, z)): export_units[frame][jnt_index] = {"translate": [_x, _y, _z]} viz.show3Dpose(p3d, ax, lcolor="#9b59b6", rcolor="#2ecc71") pngName = 'png/pose_frame_{0}.png'.format(str(frame).zfill(12)) plt.savefig(pngName) if FLAGS.write_gif: png_lib.append(imageio.imread(pngName)) # if FLAGS.cache_on_fail: # before_pose = poses3d if FLAGS.write_gif: if FLAGS.interpolation: # take every frame on gif_fps * multiplier_inv png_lib = np.array([ png_lib[png_image] for png_image in range(0, len(png_lib), int(multiplier_inv)) ]) logger.info("creating Gif gif_output/animation.gif, please Wait!") imageio.mimsave('gif_output/animation.gif', png_lib, fps=FLAGS.gif_fps) _out_file = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'maya/3d_data.json') twod_out_file = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'maya/2d_data.json') with open(_out_file, 'w') as outfile: logger.info("exported maya json to {0}".format(_out_file)) json.dump(export_units, outfile) with open(twod_out_file, 'w') as outfile: logger.info("exported maya json to {0}".format(twod_out_file)) json.dump(twod_export_units, outfile) logger.info("Done!".format(pngName))