def perform_noisy_actions(train_set_noisy, train_set_2d_org, camera_frame,
rcams, predict_14, percent):
#validate(train_set, train_set)
print('Performing noisy actions now..')
train_set_3d_noisy = addNoiseTo3D(train_set_noisy, percent)
#got the 3d data and now do 2d things
train_set_2d_noisy = du.project_to_cameras(train_set_3d_noisy, rcams)
#there can be augmented 2d data
train_set_2d_org.update(train_set_2d_noisy)
# Compute normalization statistics.
complete_train = copy.deepcopy(np.vstack(train_set_2d_org.values()))
data_mean, data_std, dim_to_ignore, dim_to_use = du.normalization_stats(
complete_train, dim=2)
# Divide every dimension independently
train_set_2d_org = du.normalize_data(train_set_2d_org, data_mean, data_std,
dim_to_use)
print("For noisy, mean :")
print(data_mean)
print("For noisy, std :")
print(data_std)
print('noisy actions done..')
return train_set_2d_org, data_mean, data_std
def read_all_data(self, actions, data_dir, one_hot=False):
"""
Loads data for training/testing and normalizes it.
Args
actions: list of strings (actions) to load
seq_length_in: number of frames to use in the burn-in sequence
seq_length_out: number of frames to use in the output sequence
data_dir: directory to load the data from
one_hot: whether to use one-hot encoding per action
Returns
train_set: dictionary with normalized training data
test_set: dictionary with test data
data_mean: d-long vector with the mean of the training data
data_std: d-long vector with the standard dev of the training data
dim_to_ignore: dimensions that are not used becaused stdev is too small
dim_to_use: dimensions that we are actually using in the model
"""
train_subject_ids = [1, 6, 7, 8, 9, 11]
test_subject_ids = [5]
train_set, complete_train = data_utils.load_data(
data_dir, train_subject_ids, actions, one_hot)
test_set, complete_test = data_utils.load_data(data_dir,
test_subject_ids,
actions, one_hot)
# Compute normalization stats
data_mean, data_std, dim_to_ignore, dim_to_use = data_utils.normalization_stats(
complete_train)
# Normalize -- subtract mean, divide by stdev
train_set = data_utils.normalize_data(train_set, data_mean, data_std,
dim_to_use, actions, one_hot)
test_set = data_utils.normalize_data(test_set, data_mean, data_std,
dim_to_use, actions, one_hot)
print("done reading data.")
self.train_set = train_set
self.test_set = test_set
self.data_mean = data_mean
self.data_std = data_std
self.dim_to_ignore = dim_to_ignore
self.dim_to_use = dim_to_use
self.train_keys = list(self.train_set.keys())
def read_all_data(actions=walking_lst,
seq_length_in=50,
seq_length_out=25,
data_dir="./data/h3.6m/dataset",
one_hot=True):
"""
Loads data for training/testing and normalizes it.
Args
actions: list of strings (actions) to load
seq_length_in: number of frames to use in the burn-in sequence
seq_length_out: number of frames to use in the output sequence
data_dir: directory to load the data from
one_hot: whether to use one-hot encoding per action
Returns
train_set: dictionary with normalized training data
test_set: dictionary with test data
data_mean: d-long vector with the mean of the training data
data_std: d-long vector with the standard dev of the training data
dim_to_ignore: dimensions that are not used becaused stdev is too small
dim_to_use: dimensions that we are actually using in the model
"""
# === Read training data ===
print("Reading training data (seq_len_in: {0}, seq_len_out {1}).".format(
seq_length_in, seq_length_out))
train_subject_ids = [1, 6, 7, 8, 9, 11]
test_subject_ids = [5]
train_set, complete_train = data_utils.load_data(data_dir,
train_subject_ids,
actions, one_hot)
test_set, complete_test = data_utils.load_data(data_dir, test_subject_ids,
actions, one_hot)
# Compute normalization stats
data_mean, data_std, dim_to_ignore, dim_to_use = data_utils.normalization_stats(
complete_train)
# Normalize -- subtract mean, divide by stdev
train_set = data_utils.normalize_data(train_set, data_mean, data_std,
dim_to_use, actions, one_hot)
test_set = data_utils.normalize_data(test_set, data_mean, data_std,
dim_to_use, actions, one_hot)
print("done reading data.")
return train_set, test_set, data_mean, data_std, dim_to_ignore, dim_to_use
poses = h5f['poses'][:]
print(poses.shape)
poses = poses[:,
SH_TO_GT_PERM, :] #按照H36M的顺序来排列stacked hourglass的数据，这一步之后就已经是h3.6m的坐标系
poses = np.reshape(poses, [poses.shape[0], -1])
print(poses.shape)
poses_final = np.zeros([poses.shape[0],
len(H36M_NAMES) * 2]) #final的size是1612*64
#*2是因为每个坐标的xy都放在一起即(x1,y1,x2,y2...)，且数据是二维的
dim_to_use_x = np.where(
np.array([x != '' and x != 'Neck/Nose' for x in H36M_NAMES]))[0] * 2
print(dim_to_use_x)
dim_to_use_y = dim_to_use_x + 1
dim_to_use = np.zeros(len(SH_NAMES) * 2, dtype=np.int32) #size为32
dim_to_use[0::2] = dim_to_use_x
print(dim_to_use)
dim_to_use[1::2] = dim_to_use_y
print(dim_to_use)
poses_final[:, dim_to_use] = poses #将pose填入poses_final当中
print(poses_final.shape)
data_mean, data_std, dim_to_ignore, dim_to_use = data_utils.normalization_stats(
poses_final, dim=2)
print(data_mean.shape, data_std.shape, dim_to_ignore, dim_to_use)
data = poses_final[:, dim_to_use]
mu = data_mean[dim_to_use]
stddev = data_std[dim_to_use]
data_out = np.divide((data - mu), stddev)
print(data_out.shape)
def read_all_data(actions, data_dir, one_hot, GT=False):
"""
Loads data for training/testing and normalizes it.
Args
Returns
"""
# === Read training data ===
print("Reading training data")
#####
#need to add one-hot vector to each frame later
#####
for i, action in enumerate(actions):
action_seq = np.load(data_dir + '/' + action + '.npz')['seq']
action_seq_len = np.load(data_dir + '/' + action + '.npz')['seq_len']
all_seq = np.concatenate(
(all_seq, action_seq), axis=0) if i > 0 else action_seq
all_seq_len = np.concatenate((all_seq_len, action_seq_len), axis=0) \
if i > 0 else action_seq_len
for i in range(len(all_seq)):
if all_seq[i, 0, 0, 0, 0] > all_seq[i, 1, 0, 0, 0]:
all_seq[i, :, :, :, 1] = -all_seq[i, :, :, :, 1]
"""
all_seq: [data_index, skeleton_num, frame_len, joints, xyz]
"""
all_seq = data_utils.rotate_data(all_seq)
shape = all_seq.shape
all_seq = np.reshape(all_seq, [shape[0], shape[1], shape[2], -1])
np.savez('val.npz', seq=all_seq)
from sklearn.model_selection import train_test_split
train_seq, test_seq, train_seq_len, test_seq_len = train_test_split(
all_seq, all_seq_len, test_size=0.25, random_state=1)
if GT:
return test_seq
for i, (seq, seq_len) in enumerate(zip(train_seq, train_seq_len)):
try:
complete_seq = np.concatenate((complete_seq, seq[: ,:int(seq_len)]), axis=1) if i > 0 \
else seq[:, :int(seq_len)]
except TypeError:
import pdb
pdb.set_trace()
complete_real_person, complete_character = complete_seq
'''
#for data_visualization
np.savez('all_data_mirror.npz',
real_person=complete_real_person,
character=complete_character,
ground_truth=complete_character,
loss=0)
sys.exit()
'''
# Compute normalization stats
rp_stats = data_utils.normalization_stats(complete_real_person,
FLAGS.dim_to_compressed)
ch_stats = data_utils.normalization_stats(complete_character,
FLAGS.dim_to_compressed)
# Normalize -- subtract mean, divide by stdev
train_shape = train_seq.shape
test_shape = test_seq.shape
if rp_stats[4] is not None:
normalized_train_seq = np.zeros([
train_shape[0], train_shape[1], train_shape[2],
rp_stats[4].get_params()['n_components']
])
normalized_test_seq = np.zeros([
test_shape[0], test_shape[1], test_shape[2],
rp_stats[4].get_params()['n_components']
])
else:
normalized_train_seq = np.zeros(train_shape)
normalized_test_seq = np.zeros(test_shape)
normalized_train_seq[:, 0] = data_utils.normalize_data(
train_seq[:, 0], rp_stats, actions, one_hot)
normalized_train_seq[:, 1] = data_utils.normalize_data(
train_seq[:, 1], ch_stats, actions, one_hot)
normalized_test_seq[:,
0] = data_utils.normalize_data(test_seq[:,
0], rp_stats,
actions, one_hot)
normalized_test_seq[:,
1] = data_utils.normalize_data(test_seq[:,
1], ch_stats,
actions, one_hot)
print("done reading data.")
return normalized_train_seq, normalized_test_seq, train_seq_len, test_seq_len, rp_stats, ch_stats, \
max(all_seq_len)
import data_utils
from viz import Ax3DPose
data_dir = './data/h36m/dataset'
train_subject_ids = [1, 6, 7, 8, 9, 11]
test_subject_ids = [5]
actions = [
"walking", "eating", "smoking", "discussion", "directions", "greeting",
"phoning", "posing", "purchases", "sitting", "sittingdown", "takingphoto",
"waiting", "walkingdog", "walkingtogether"
]
one_hot = True
train_set, complete_train = data_utils.load_data(data_dir, train_subject_ids,
actions, one_hot)
test_set, complete_test = data_utils.load_data(data_dir, test_subject_ids,
actions, one_hot)
data_mean, data_std, dim_to_ignore, dim_to_use = data_utils.normalization_stats(
complete_train)
train_set = data_utils.normalize_data(train_set, data_mean, data_std,
dim_to_use, actions, one_hot)
test_set = data_utils.normalize_data(test_set, data_mean, data_std, dim_to_use,
actions, one_hot)
print("done reading data.")
print(train_set[(1, "walking", 1, "even")])
def __init__(self, sampling=True):
# def create_model(actions, sampling=False):
# Learning
self.seq_out = 25
parser = argparse.ArgumentParser(
description='Train RNN for human pose estimation')
parser.add_argument('--learning_rate',
dest='learning_rate',
help='Learning rate',
default=0.005,
type=float)
parser.add_argument(
'--learning_rate_decay_factor',
dest='learning_rate_decay_factor',
help='Learning rate is multiplied by this much. 1 means no decay.',
default=0.95,
type=float)
parser.add_argument('--learning_rate_step',
dest='learning_rate_step',
help='Every this many steps, do decay.',
default=10000,
type=int)
parser.add_argument('--batch_size',
dest='batch_size',
help='Batch size to use during training.',
default=16,
type=int)
parser.add_argument('--max_gradient_norm',
dest='max_gradient_norm',
help='Clip gradients to this norm.',
default=5,
type=float)
parser.add_argument('--iterations',
dest='iterations',
help='Iterations to train for.',
default=1e5,
type=int)
parser.add_argument('--test_every',
dest='test_every',
help='',
default=100,
type=int)
# Architecture
parser.add_argument('--architecture',
dest='architecture',
help='Seq2seq architecture to use: [basic, tied].',
default='tied',
type=str)
parser.add_argument(
'--loss_to_use',
dest='loss_to_use',
help='The type of loss to use, supervised or sampling_based',
default='sampling_based',
type=str)
parser.add_argument(
'--residual_velocities',
dest='residual_velocities',
help='Add a residual connection that effectively models velocities',
action='store_true',
default=False)
parser.add_argument('--size',
dest='size',
help='Size of each model layer.',
default=1024,
type=int)
parser.add_argument('--num_layers',
dest='num_layers',
help='Number of layers in the model.',
default=1,
type=int)
parser.add_argument(
'--seq_length_in',
dest='seq_length_in',
help='Number of frames to feed into the encoder. 25 fp',
default=50,
type=int)
parser.add_argument(
'--seq_length_out',
dest='seq_length_out',
help='Number of frames that the decoder has to predict. 25fps',
default=self.seq_out,
type=int)
parser.add_argument('--omit_one_hot',
dest='omit_one_hot',
help='',
action='store_true',
default=False)
# Directories
parser.add_argument(
'--data_dir',
dest='data_dir',
help='Data directory',
default=os.path.normpath("./data_IK/h3.6m/dataset"),
type=str)
parser.add_argument('--train_dir',
dest='train_dir',
help='Training directory',
default=os.path.normpath("./experiments/"),
type=str)
parser.add_argument(
'--action',
dest='action',
help=
'The action to train on. all means all the actions, all_periodic means walking, eating and smoking',
default="posing",
type=str)
parser.add_argument('--use_cpu',
dest='use_cpu',
help='',
action='store_true',
default=False)
# parser.add_argument('--load', dest='load',
# help='Try to load a previous checkpoint.',
# default=, type=int)
parser.add_argument('--sample',
dest='sample',
help='Set to True for sampling.',
action='store_true',
default=False)
self.args = parser.parse_args()
self.model = seq2seq_model.Seq2SeqModel(
self.args.architecture,
self.args.seq_length_in if not sampling else 50,
self.args.seq_length_out if not sampling else self.seq_out,
self.args.size, # hidden layer size
self.args.num_layers,
self.args.max_gradient_norm,
self.args.batch_size,
self.args.learning_rate,
self.args.learning_rate_decay_factor,
self.args.loss_to_use if not sampling else "sampling_based",
1,
not self.args.omit_one_hot,
self.args.residual_velocities,
dtype=torch.float32)
print("Loading model")
self.model = torch.load('./25_best_model_7100')
self.model.source_seq_len = 50
self.model.target_seq_len = self.seq_out
act = ["posing"]
train_subject_ids = [1, 6]
train_set, complete_train = data_utils.load_data(
self.args.data_dir, train_subject_ids, act,
not self.args.omit_one_hot)
self.data_mean, self.data_std, self.dim_to_ignore, self.dim_to_use = data_utils.normalization_stats(
complete_train)
def main(_):
smoothed = read_openpose_json_mydata()
plt.figure(2)
smooth_curves_plot = show_anim_curves(smoothed, plt)
# return
pngName = 'gif_output/smooth_plot.png'
smooth_curves_plot.savefig(pngName)
logger.info('writing gif_output/smooth_plot.png')
data_dir = '/home/lyunfan/NYU_summer_intern/3d_pose_baseline_pytorch/data/'
stat_3d = torch.load(os.path.join(data_dir, 'stat_3d.pth.tar'))
train_2d = torch.load(os.path.join(data_dir, 'train_2d_ft.pth.tar'))
complete_train = copy.deepcopy(np.vstack(train_2d.values()))
data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.normalization_stats(
complete_train, dim=2)
data_mean_3d, data_std_3d, dim_to_ignore_3d, dim_to_use_3d = stat_3d['mean'], stat_3d['std'], stat_3d['dim_ignore'], \
stat_3d['dim_use'],
if FLAGS.interpolation:
logger.info("start interpolation")
framerange = len(smoothed.keys())
joint_rows = 36
array = np.concatenate(list(smoothed.values()))
array_reshaped = np.reshape(array, (framerange, joint_rows))
multiplier = FLAGS.multiplier
multiplier_inv = 1 / multiplier
out_array = np.array([])
for row in range(joint_rows):
x = []
for frame in range(framerange):
x.append(array_reshaped[frame, row])
frame = range(framerange)
frame_resampled = np.arange(0, framerange, multiplier)
spl = UnivariateSpline(frame, x, k=3)
# relative smooth factor based on jnt anim curve
min_x, max_x = min(x), max(x)
smooth_fac = max_x - min_x
smooth_resamp = 125
smooth_fac = smooth_fac * smooth_resamp
spl.set_smoothing_factor(float(smooth_fac))
xnew = spl(frame_resampled)
out_array = np.append(out_array, xnew)
logger.info(
"done interpolating. reshaping {0} frames, please wait!!".format(
framerange))
a = np.array([])
for frame in range(int(framerange * multiplier_inv)):
jnt_array = []
for jnt in range(joint_rows):
jnt_array.append(
out_array[jnt * int(framerange * multiplier_inv) + frame])
a = np.append(a, jnt_array)
a = np.reshape(a, (int(framerange * multiplier_inv), joint_rows))
out_array = a
interpolate_smoothed = {}
for frame in range(int(framerange * multiplier_inv)):
interpolate_smoothed[frame] = list(out_array[frame])
plt.figure(3)
smoothed = interpolate_smoothed
interpolate_curves_plot = show_anim_curves(smoothed, plt)
pngName = 'gif_output/interpolate_{0}.png'.format(smooth_resamp)
interpolate_curves_plot.savefig(pngName)
logger.info('writing gif_output/interpolate_plot.png')
enc_in = np.zeros((1, 64))
enc_in[0] = [0 for i in range(64)]
actions = data_utils.define_actions(FLAGS.action)
# SUBJECT_IDS = [1, 5, 6, 7, 8, 9, 11]
# rcams = cameras.load_cameras(FLAGS.cameras_path, SUBJECT_IDS)
# train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.read_2d_predictions(
# actions, FLAGS.data_dir)
# train_set_3d, test_set_3d, data_mean_3d, data_std_3d, dim_to_ignore_3d, dim_to_use_3d, train_root_positions, test_root_positions = data_utils.read_3d_data(
# actions, FLAGS.data_dir, FLAGS.camera_frame, rcams, FLAGS.predict_14)
device_count = {"GPU": 1}
png_lib = []
before_pose = None
with tf.Session(config=tf.ConfigProto(device_count=device_count,
allow_soft_placement=True)) as sess:
# plt.figure(3)
batch_size = 128
model = create_model(sess, actions, batch_size)
iter_range = len(smoothed.keys())
export_units = {}
twod_export_units = {}
for n, (frame, xy) in enumerate(smoothed.items()):
logger.info("calc frame {0}/{1}".format(frame, iter_range))
# map list into np array
# joints_array = np.zeros((1, 36))
# joints_array[0] = [0 for i in range(36)]
# for o in range(len(joints_array[0])):
# # feed array with xy array
# joints_array[0][o] = float(xy[o])
# _data = joints_array[0]
# NEWLY ADDED
enc_in = np.zeros((1, 64))
enc_in[0] = [0 for i in range(64)]
twod_export_units[frame] = {}
for abs_b, __n in enumerate(range(0, len(xy), 2)):
twod_export_units[frame][abs_b] = {
"translate": [xy[__n], xy[__n + 1]]
}
_data = [float(i) for i in xy][:36]
# mapping all body parts or 3d-pose-baseline format
for i in range(len(order)):
for j in range(2):
# create encoder input
enc_in[0][order[i] * 2 + j] = _data[i * 2 + j]
for j in range(2):
# Hip
enc_in[0][0 * 2 + j] = (enc_in[0][1 * 2 + j] +
enc_in[0][6 * 2 + j]) / 2
# Neck/Nose
enc_in[0][14 * 2 + j] = (enc_in[0][15 * 2 + j] +
enc_in[0][12 * 2 + j]) / 2
# Thorax
enc_in[0][13 * 2 +
j] = 2 * enc_in[0][12 * 2 + j] - enc_in[0][14 * 2 +
j]
# set spine
spine_x = enc_in[0][24]
spine_y = enc_in[0][25]
enc_in = enc_in[:, dim_to_use_2d]
# mu = data_mean_2d[dim_to_use_2d]
mu = data_mean_2d
# stddev = data_std_2d[dim_to_use_2d]
stddev = data_std_2d
enc_in = np.divide((enc_in - mu), stddev)
dp = 1.0
dec_out = np.zeros((1, 48))
dec_out[0] = [0 for i in range(48)]
_, _, poses3d = model.step(sess,
enc_in,
dec_out,
dp,
isTraining=False)
all_poses_3d = []
enc_in = data_utils.unNormalizeData2d(enc_in, data_mean_2d,
data_std_2d, dim_to_use_2d)
poses3d = data_utils.unNormalizeData(poses3d, data_mean_3d,
data_std_3d, dim_to_ignore_3d)
gs1 = gridspec.GridSpec(1, 1)
gs1.update(wspace=-0.00,
hspace=0.05) # set the spacing between axes.
plt.axis('off')
all_poses_3d.append(poses3d)
enc_in, poses3d = map(np.vstack, [enc_in, all_poses_3d])
subplot_idx, exidx = 1, 1
_max = 0
_min = 10000
for i in range(poses3d.shape[0]):
for j in range(32):
tmp = poses3d[i][j * 3 + 2]
poses3d[i][j * 3 + 2] = poses3d[i][j * 3 + 1]
poses3d[i][j * 3 + 1] = tmp
if poses3d[i][j * 3 + 2] > _max:
_max = poses3d[i][j * 3 + 2]
if poses3d[i][j * 3 + 2] < _min:
_min = poses3d[i][j * 3 + 2]
for i in range(poses3d.shape[0]):
for j in range(32):
poses3d[i][j * 3 + 2] = _max - poses3d[i][j * 3 + 2] + _min
poses3d[i][j * 3] += (spine_x - 630)
poses3d[i][j * 3 + 2] += (500 - spine_y)
# Plot 3d predictions
ax = plt.subplot(gs1[subplot_idx - 1], projection='3d')
ax.view_init(18, -70)
# if FLAGS.cache_on_fail:
# if np.min(poses3d) < -1000 and before_pose:
# poses3d = before_pose
p3d = poses3d
logger.info("frame score {0}".format(np.min(poses3d)))
x, y, z = [[] for _ in range(3)]
if not poses3d is None:
to_export = poses3d.tolist()[0]
else:
to_export = [0.0 for _ in range(96)]
logger.info("export {0}".format(to_export))
for o in range(0, len(to_export), 3):
x.append(to_export[o])
y.append(to_export[o + 1])
z.append(to_export[o + 2])
export_units[frame] = {}
for jnt_index, (_x, _y, _z) in enumerate(zip(x, y, z)):
export_units[frame][jnt_index] = {"translate": [_x, _y, _z]}
viz.show3Dpose(p3d, ax, lcolor="#9b59b6", rcolor="#2ecc71")
pngName = 'png/pose_frame_{0}.png'.format(str(frame).zfill(12))
plt.savefig(pngName)
if FLAGS.write_gif:
png_lib.append(imageio.imread(pngName))
# if FLAGS.cache_on_fail:
# before_pose = poses3d
if FLAGS.write_gif:
if FLAGS.interpolation:
# take every frame on gif_fps * multiplier_inv
png_lib = np.array([
png_lib[png_image]
for png_image in range(0, len(png_lib), int(multiplier_inv))
])
logger.info("creating Gif gif_output/animation.gif, please Wait!")
imageio.mimsave('gif_output/animation.gif', png_lib, fps=FLAGS.gif_fps)
_out_file = os.path.join(os.path.dirname(os.path.dirname(__file__)),
'maya/3d_data.json')
twod_out_file = os.path.join(os.path.dirname(os.path.dirname(__file__)),
'maya/2d_data.json')
with open(_out_file, 'w') as outfile:
logger.info("exported maya json to {0}".format(_out_file))
json.dump(export_units, outfile)
with open(twod_out_file, 'w') as outfile:
logger.info("exported maya json to {0}".format(twod_out_file))
json.dump(twod_export_units, outfile)
logger.info("Done!".format(pngName))
