def __init__(self, use_gpu=False):
actions = data_utils.define_actions(FLAGS.action)
rcams = cameras.load_cameras(FLAGS.cameras_path, SUBJECT_IDS)
train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.read_2d_predictions(
actions, FLAGS.data_dir)
train_set_3d, test_set_3d, data_mean_3d, data_std_3d, dim_to_ignore_3d, dim_to_use_3d, train_root_positions, test_root_positions = data_utils.read_3d_data(
actions, FLAGS.data_dir, FLAGS.camera_frame, rcams,
FLAGS.predict_14)
self.train_set_2d = train_set_2d
self.test_set_2d = test_set_2d
self.data_mean_2d = data_mean_2d
self.data_std_2d = data_std_2d
self.dim_to_use_2d = dim_to_use_2d
self.dim_to_ignore_2d = dim_to_ignore_2d
self.train_set_3d = train_set_3d
self.test_set_3d = test_set_3d
self.data_mean_3d = data_mean_3d
self.data_std_3d = data_std_3d
self.dim_to_use_3d = dim_to_use_3d
self.dim_to_ignore_3d = dim_to_ignore_3d
device_count = {"GPU": 1} if use_gpu else {"GPU": 0}
self.persistent_sess = tf.Session(config=tf.ConfigProto(
device_count=device_count, allow_soft_placement=True))
with self.persistent_sess.as_default():
self.graph = tf.get_default_graph()
self.model = create_model(self.persistent_sess, actions,
batch_size)
def main(_):
#ABS_DIR = os.path.abspath('.')
posf = open(pose_output_dir, 'w')
#smoothedf = open(ABS_DIR + '/tmp/smoothed.txt', 'w')
smoothed = read_openpose_json()
plt.figure(2)
smooth_curves_plot = show_anim_curves(smoothed, plt)
pngName = 'gif_output/smooth_plot.png'
smooth_curves_plot.savefig(pngName)
logger.info('writing gif_output/smooth_plot.png')
if FLAGS.interpolation:
logger.info("start interpolation")
framerange = len( smoothed.keys() )
joint_rows = 36
array = np.concatenate(list(smoothed.values()))
array_reshaped = np.reshape(array, (framerange, joint_rows) )
print(array_reshaped[0,:])
arm = [4,5,6,7,8,9,10,11]
multiplier = FLAGS.multiplier
multiplier_inv = 1/multiplier
out_array = np.array([])
for row in range(joint_rows):
x = []
for frame in range(framerange):
x.append( array_reshaped[frame, row] )
frame = range( framerange )
frame_resampled = np.arange(0, framerange, multiplier)
spl = UnivariateSpline(frame, x, k=3)
#relative smooth factor based on jnt anim curve
min_x, max_x = min(x), max(x)
smooth_fac = max_x - min_x
if row in arm:
smooth_resamp = 1
else:
smooth_resamp = 75
smooth_fac = smooth_fac * smooth_resamp
spl.set_smoothing_factor( float(smooth_fac) )
xnew = spl(frame_resampled)
out_array = np.append(out_array, xnew)
logger.info("done interpolating. reshaping {0} frames, please wait!!".format(framerange))
a = np.array([])
for frame in range( int( framerange * multiplier_inv ) ):
jnt_array = []
for jnt in range(joint_rows):
jnt_array.append( out_array[ jnt * int(framerange * multiplier_inv) + frame] )
a = np.append(a, jnt_array)
a = np.reshape(a, (int(framerange * multiplier_inv), joint_rows))
out_array = a
interpolate_smoothed = {}
for frame in range( int(framerange * multiplier_inv) ):
interpolate_smoothed[frame] = list( out_array[frame] )
plt.figure(3)
smoothed = interpolate_smoothed
interpolate_curves_plot = show_anim_curves(smoothed, plt)
pngName = 'gif_output/interpolate_{0}.png'.format(smooth_resamp)
interpolate_curves_plot.savefig(pngName)
logger.info('writing gif_output/interpolate_plot.png')
enc_in = np.zeros((1, 64))
enc_in[0] = [0 for i in range(64)]
actions = data_utils.define_actions(FLAGS.action)
SUBJECT_IDS = [1, 5, 6, 7, 8, 9, 11]
rcams = cameras.load_cameras(FLAGS.cameras_path, SUBJECT_IDS)
train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.read_2d_predictions(
actions, FLAGS.data_dir)
train_set_3d, test_set_3d, data_mean_3d, data_std_3d, dim_to_ignore_3d, dim_to_use_3d, train_root_positions, test_root_positions = data_utils.read_3d_data(
actions, FLAGS.data_dir, FLAGS.camera_frame, rcams, FLAGS.predict_14)
device_count = {"GPU": 1}
png_lib = []
with tf.Session(config=tf.ConfigProto(
device_count=device_count,
allow_soft_placement=True)) as sess:
#plt.figure(3)
batch_size = 128
model = create_model(sess, actions, batch_size)
iter_range = len(smoothed.keys())
for n, (frame, xy) in enumerate(smoothed.items()):
logger.info("calc frame {0}/{1}".format(frame, iter_range))
# map list into np array
joints_array = np.zeros((1, 36))
joints_array[0] = [0 for i in range(36)]
for o in range(len(joints_array[0])):
#feed array with xy array
joints_array[0][o] = xy[o]
_data = joints_array[0]
#smoothedf.write(' '.join(map(str, _data)))
#smoothedf.write("\n")
# mapping all body parts or 3d-pose-baseline format
for i in range(len(order)):
for j in range(2):
# create encoder input
enc_in[0][order[i] * 2 + j] = _data[i * 2 + j]
for j in range(2):
# Hip
enc_in[0][0 * 2 + j] = (enc_in[0][1 * 2 + j] + enc_in[0][6 * 2 + j]) / 2
# Neck/Nose
enc_in[0][14 * 2 + j] = (enc_in[0][15 * 2 + j] + enc_in[0][12 * 2 + j]) / 2
# Thorax
enc_in[0][13 * 2 + j] = 2 * enc_in[0][12 * 2 + j] - enc_in[0][14 * 2 + j]
# set spine
spine_x = enc_in[0][24]
spine_y = enc_in[0][25]
enc_in = enc_in[:, dim_to_use_2d]
mu = data_mean_2d[dim_to_use_2d]
stddev = data_std_2d[dim_to_use_2d]
enc_in = np.divide((enc_in - mu), stddev)
dp = 1.0
dec_out = np.zeros((1, 48))
dec_out[0] = [0 for i in range(48)]
_, _, poses3d = model.step(sess, enc_in, dec_out, dp, isTraining=False)
all_poses_3d = []
enc_in = data_utils.unNormalizeData(enc_in, data_mean_2d, data_std_2d, dim_to_ignore_2d)
poses3d = data_utils.unNormalizeData(poses3d, data_mean_3d, data_std_3d, dim_to_ignore_3d)
gs1 = gridspec.GridSpec(1, 1)
gs1.update(wspace=-0.00, hspace=0.05) # set the spacing between axes.
plt.axis('off')
all_poses_3d.append( poses3d )
enc_in, poses3d = map( np.vstack, [enc_in, all_poses_3d] )
subplot_idx, exidx = 1, 1
_max = 0
_min = 10000
for i in range(poses3d.shape[0]):
for j in range(32):
tmp = poses3d[i][j * 3 + 2]
poses3d[i][j * 3 + 2] = poses3d[i][j * 3 + 1]
poses3d[i][j * 3 + 1] = tmp
if poses3d[i][j * 3 + 2] > _max:
_max = poses3d[i][j * 3 + 2]
if poses3d[i][j * 3 + 2] < _min:
_min = poses3d[i][j * 3 + 2]
for i in range(poses3d.shape[0]):
for j in range(32):
poses3d[i][j * 3 + 2] = _max - poses3d[i][j * 3 + 2] + _min
poses3d[i][j * 3] += (spine_x - 630)
poses3d[i][j * 3 + 2] += (500 - spine_y)
# Plot 3d predictions
ax = plt.subplot(gs1[subplot_idx - 1], projection='3d')
ax.view_init(18, -70)
if np.min(poses3d) < -1000:
try:
poses3d = before_pose
except:
pass
p3d = poses3d
#viz.show3Dpose(p3d, ax, lcolor="#9b59b6", rcolor="#2ecc71")
# pngName = 'png/pose_frame_{0}.png'.format(str(frame).zfill(12))
# plt.savefig(pngName)
# if FLAGS.write_gif:
# png_lib.append(imageio.imread(pngName))
before_pose = poses3d
write_pos_data(poses3d, ax, posf)
posf.close()
def main(_):
smoothed = read_openpose_json()
plt.figure(2)
smooth_curves_plot = show_anim_curves(smoothed, plt)
#return
pngName = 'gif_output/smooth_plot.png'
smooth_curves_plot.savefig(pngName)
logger.info('writing gif_output/smooth_plot.png')
if FLAGS.interpolation:
logger.info("start interpolation")
framerange = len( smoothed.keys() )
joint_rows = 36
array = np.concatenate(list(smoothed.values()))
array_reshaped = np.reshape(array, (framerange, joint_rows) )
multiplier = FLAGS.multiplier
multiplier_inv = 1/multiplier
out_array = np.array([])
for row in range(joint_rows):
x = []
for frame in range(framerange):
x.append( array_reshaped[frame, row] )
frame = range( framerange )
frame_resampled = np.arange(0, framerange, multiplier)
spl = UnivariateSpline(frame, x, k=3)
#relative smooth factor based on jnt anim curve
min_x, max_x = min(x), max(x)
smooth_fac = max_x - min_x
smooth_resamp = 125
smooth_fac = smooth_fac * smooth_resamp
spl.set_smoothing_factor( float(smooth_fac) )
xnew = spl(frame_resampled)
out_array = np.append(out_array, xnew)
logger.info("done interpolating. reshaping {0} frames, please wait!!".format(framerange))
a = np.array([])
for frame in range( int( framerange * multiplier_inv ) ):
jnt_array = []
for jnt in range(joint_rows):
jnt_array.append( out_array[ jnt * int(framerange * multiplier_inv) + frame] )
a = np.append(a, jnt_array)
a = np.reshape(a, (int(framerange * multiplier_inv), joint_rows))
out_array = a
interpolate_smoothed = {}
for frame in range( int(framerange * multiplier_inv) ):
interpolate_smoothed[frame] = list( out_array[frame] )
plt.figure(3)
smoothed = interpolate_smoothed
interpolate_curves_plot = show_anim_curves(smoothed, plt)
pngName = 'gif_output/interpolate_{0}.png'.format(smooth_resamp)
interpolate_curves_plot.savefig(pngName)
logger.info('writing gif_output/interpolate_plot.png')
enc_in = np.zeros((1, 64))
enc_in[0] = [0 for i in range(64)]
actions = data_utils.define_actions(FLAGS.action)
SUBJECT_IDS = [1, 5, 6, 7, 8, 9, 11]
rcams = cameras.load_cameras(FLAGS.cameras_path, SUBJECT_IDS)
train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.read_2d_predictions(
actions, FLAGS.data_dir)
train_set_3d, test_set_3d, data_mean_3d, data_std_3d, dim_to_ignore_3d, dim_to_use_3d, train_root_positions, test_root_positions = data_utils.read_3d_data(
actions, FLAGS.data_dir, FLAGS.camera_frame, rcams, FLAGS.predict_14)
device_count = {"GPU": 1}
png_lib = []
before_pose = None
with tf.Session(config=tf.ConfigProto(
device_count=device_count,
allow_soft_placement=True)) as sess:
#plt.figure(3)
batch_size = 128
model = create_model(sess, actions, batch_size)
iter_range = len(smoothed.keys())
export_units = {}
twod_export_units = {}
for n, (frame, xy) in enumerate(smoothed.items()):
logger.info("calc frame {0}/{1}".format(frame, iter_range))
# map list into np array
joints_array = np.zeros((1, 36))
joints_array[0] = [0 for i in range(36)]
for o in range(len(joints_array[0])):
#feed array with xy array
joints_array[0][o] = xy[o]
twod_export_units[frame]={}
for abs_b, __n in enumerate(range(0, len(xy),2)):
twod_export_units[frame][abs_b] = {"translate": [xy[__n],xy[__n+1]]}
_data = joints_array[0]
# mapping all body parts or 3d-pose-baseline format
for i in range(len(order)):
for j in range(2):
# create encoder input
enc_in[0][order[i] * 2 + j] = _data[i * 2 + j]
for j in range(2):
# Hip
enc_in[0][0 * 2 + j] = (enc_in[0][1 * 2 + j] + enc_in[0][6 * 2 + j]) / 2
# Neck/Nose
enc_in[0][14 * 2 + j] = (enc_in[0][15 * 2 + j] + enc_in[0][12 * 2 + j]) / 2
# Thorax
enc_in[0][13 * 2 + j] = 2 * enc_in[0][12 * 2 + j] - enc_in[0][14 * 2 + j]
# set spine
spine_x = enc_in[0][24]
spine_y = enc_in[0][25]
enc_in = enc_in[:, dim_to_use_2d]
mu = data_mean_2d[dim_to_use_2d]
stddev = data_std_2d[dim_to_use_2d]
enc_in = np.divide((enc_in - mu), stddev)
dp = 1.0
dec_out = np.zeros((1, 48))
dec_out[0] = [0 for i in range(48)]
_, _, poses3d = model.step(sess, enc_in, dec_out, dp, isTraining=False)
all_poses_3d = []
enc_in = data_utils.unNormalizeData(enc_in, data_mean_2d, data_std_2d, dim_to_ignore_2d)
poses3d = data_utils.unNormalizeData(poses3d, data_mean_3d, data_std_3d, dim_to_ignore_3d)
gs1 = gridspec.GridSpec(1, 1)
gs1.update(wspace=-0.00, hspace=0.05) # set the spacing between axes.
plt.axis('off')
all_poses_3d.append( poses3d )
enc_in, poses3d = map( np.vstack, [enc_in, all_poses_3d] )
subplot_idx, exidx = 1, 1
_max = 0
_min = 10000
for i in range(poses3d.shape[0]):
for j in range(32):
tmp = poses3d[i][j * 3 + 2]
poses3d[i][j * 3 + 2] = poses3d[i][j * 3 + 1]
poses3d[i][j * 3 + 1] = tmp
if poses3d[i][j * 3 + 2] > _max:
_max = poses3d[i][j * 3 + 2]
if poses3d[i][j * 3 + 2] < _min:
_min = poses3d[i][j * 3 + 2]
for i in range(poses3d.shape[0]):
for j in range(32):
poses3d[i][j * 3 + 2] = _max - poses3d[i][j * 3 + 2] + _min
poses3d[i][j * 3] += (spine_x - 630)
poses3d[i][j * 3 + 2] += (500 - spine_y)
# Plot 3d predictions
ax = plt.subplot(gs1[subplot_idx - 1], projection='3d')
ax.view_init(18, -70)
if FLAGS.cache_on_fail:
if np.min(poses3d) < -1000:
poses3d = before_pose
p3d = poses3d
to_export = poses3d.tolist()[0]
x,y,z = [[] for _ in range(3)]
for o in range(0, len(to_export), 3):
x.append(to_export[o])
y.append(to_export[o+1])
z.append(to_export[o+2])
export_units[frame]={}
for jnt_index, (_x, _y, _z) in enumerate(zip(x,y,z)):
export_units[frame][jnt_index] = {"translate": [_x, _y, _z]}
viz.show3Dpose(p3d, ax, lcolor="#9b59b6", rcolor="#2ecc71")
pngName = 'png/pose_frame_{0}.png'.format(str(frame).zfill(12))
plt.savefig(pngName)
if FLAGS.write_gif:
png_lib.append(imageio.imread(pngName))
if FLAGS.cache_on_fail:
before_pose = poses3d
if FLAGS.write_gif:
if FLAGS.interpolation:
#take every frame on gif_fps * multiplier_inv
png_lib = np.array([png_lib[png_image] for png_image in range(0,len(png_lib), int(multiplier_inv)) ])
logger.info("creating Gif gif_output/animation.gif, please Wait!")
imageio.mimsave('gif_output/animation.gif', png_lib, fps=FLAGS.gif_fps)
_out_file = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'maya/3d_data.json')
twod_out_file = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'maya/2d_data.json')
with open(_out_file, 'w') as outfile:
logger.info("exported maya json to {0}".format(_out_file))
json.dump(export_units, outfile)
with open(twod_out_file, 'w') as outfile:
logger.info("exported maya json to {0}".format(twod_out_file))
json.dump(twod_export_units, outfile)
logger.info("Done!".format(pngName))
def main(_):
# 出力用日付
now_str = "{0:%Y%m%d_%H%M%S}".format(datetime.datetime.now())
logger.debug("FLAGS.person_idx={0}".format(FLAGS.person_idx))
# 日付+indexディレクトリ作成
subdir = '{0}/{1}_3d_{2}_idx{3:02d}'.format(
os.path.dirname(openpose_output_dir),
os.path.basename(openpose_output_dir), now_str, FLAGS.person_idx)
os.makedirs(subdir)
frame3d_dir = "{0}/frame3d".format(subdir)
os.makedirs(frame3d_dir)
#関節位置情報ファイル
posf = open(subdir + '/pos.txt', 'w')
#正規化済みOpenpose位置情報ファイル
smoothedf = open(subdir + '/smoothed.txt', 'w')
idx = FLAGS.person_idx - 1
smoothed = openpose_utils.read_openpose_json(openpose_output_dir, idx,
level[FLAGS.verbose] == 3)
logger.info("reading and smoothing done. start feeding 3d-pose-baseline")
logger.debug(smoothed)
plt.figure(2)
smooth_curves_plot = show_anim_curves(smoothed, plt)
pngName = subdir + '/smooth_plot.png'
smooth_curves_plot.savefig(pngName)
enc_in = np.zeros((1, 64))
enc_in[0] = [0 for i in range(64)]
actions = data_utils.define_actions(FLAGS.action)
SUBJECT_IDS = [1, 5, 6, 7, 8, 9, 11]
rcams = cameras.load_cameras(FLAGS.cameras_path, SUBJECT_IDS)
train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.read_2d_predictions(
actions, FLAGS.data_dir)
train_set_3d, test_set_3d, data_mean_3d, data_std_3d, dim_to_ignore_3d, dim_to_use_3d, train_root_positions, test_root_positions = data_utils.read_3d_data(
actions, FLAGS.data_dir, FLAGS.camera_frame, rcams, FLAGS.predict_14)
before_pose = None
device_count = {"GPU": 1}
png_lib = []
with tf.Session(config=tf.ConfigProto(device_count=device_count,
allow_soft_placement=True)) as sess:
#plt.figure(3)
batch_size = 128
model = create_model(sess, actions, batch_size)
for n, (frame, xy) in enumerate(smoothed.items()):
logger.info("calc idx {0}, frame {1}".format(idx, frame))
# map list into np array
joints_array = np.zeros((1, 36))
joints_array[0] = [0 for i in range(36)]
for o in range(len(joints_array[0])):
#feed array with xy array
joints_array[0][o] = xy[o]
_data = joints_array[0]
smoothedf.write(' '.join(map(str, _data)))
smoothedf.write("\n")
# mapping all body parts or 3d-pose-baseline format
for i in range(len(order)):
for j in range(2):
# create encoder input
enc_in[0][order[i] * 2 + j] = _data[i * 2 + j]
for j in range(2):
# Hip
enc_in[0][0 * 2 + j] = (enc_in[0][1 * 2 + j] +
enc_in[0][6 * 2 + j]) / 2
# Neck/Nose
enc_in[0][14 * 2 + j] = (enc_in[0][15 * 2 + j] +
enc_in[0][12 * 2 + j]) / 2
# Thorax
enc_in[0][13 * 2 +
j] = 2 * enc_in[0][12 * 2 + j] - enc_in[0][14 * 2 +
j]
# set spine
spine_x = enc_in[0][24]
spine_y = enc_in[0][25]
# logger.debug("enc_in - 1")
# logger.debug(enc_in)
enc_in = enc_in[:, dim_to_use_2d]
mu = data_mean_2d[dim_to_use_2d]
stddev = data_std_2d[dim_to_use_2d]
enc_in = np.divide((enc_in - mu), stddev)
dp = 1.0
dec_out = np.zeros((1, 48))
dec_out[0] = [0 for i in range(48)]
_, _, poses3d = model.step(sess,
enc_in,
dec_out,
dp,
isTraining=False)
all_poses_3d = []
enc_in = data_utils.unNormalizeData(enc_in, data_mean_2d,
data_std_2d, dim_to_ignore_2d)
poses3d = data_utils.unNormalizeData(poses3d, data_mean_3d,
data_std_3d, dim_to_ignore_3d)
gs1 = gridspec.GridSpec(1, 1)
gs1.update(wspace=-0.00,
hspace=0.05) # set the spacing between axes.
plt.axis('off')
all_poses_3d.append(poses3d)
enc_in, poses3d = map(np.vstack, [enc_in, all_poses_3d])
subplot_idx, exidx = 1, 1
max = 0
min = 10000
# logger.debug("enc_in - 2")
# logger.debug(enc_in)
for i in range(poses3d.shape[0]):
for j in range(32):
tmp = poses3d[i][j * 3 + 2]
poses3d[i][j * 3 + 2] = poses3d[i][j * 3 + 1]
poses3d[i][j * 3 + 1] = tmp
if poses3d[i][j * 3 + 2] > max:
max = poses3d[i][j * 3 + 2]
if poses3d[i][j * 3 + 2] < min:
min = poses3d[i][j * 3 + 2]
for i in range(poses3d.shape[0]):
for j in range(32):
poses3d[i][j * 3 + 2] = max - poses3d[i][j * 3 + 2] + min
poses3d[i][j * 3] += (spine_x - 630)
poses3d[i][j * 3 + 2] += (500 - spine_y)
# Plot 3d predictions
ax = plt.subplot(gs1[subplot_idx - 1], projection='3d')
ax.view_init(18, 280)
logger.debug(np.min(poses3d))
if np.min(poses3d) < -1000 and before_pose is not None:
poses3d = before_pose
p3d = poses3d
# logger.debug("poses3d")
# logger.debug(poses3d)
if level[FLAGS.verbose] == logging.INFO:
viz.show3Dpose(p3d,
ax,
lcolor="#9b59b6",
rcolor="#2ecc71",
add_labels=True)
# 各フレームの単一視点からのはINFO時のみ
pngName = frame3d_dir + '/tmp_{0:012d}.png'.format(frame)
plt.savefig(pngName)
png_lib.append(imageio.imread(pngName))
before_pose = poses3d
# 各フレームの角度別出力はデバッグ時のみ
if level[FLAGS.verbose] == logging.DEBUG:
for azim in [0, 45, 90, 135, 180, 225, 270, 315, 360]:
ax2 = plt.subplot(gs1[subplot_idx - 1], projection='3d')
ax2.view_init(18, azim)
viz.show3Dpose(p3d,
ax2,
lcolor="#FF0000",
rcolor="#0000FF",
add_labels=True)
pngName2 = frame3d_dir + '/tmp_{0:012d}_{1:03d}.png'.format(
frame, azim)
plt.savefig(pngName2)
#関節位置情報の出力
write_pos_data(poses3d, ax, posf)
posf.close()
# INFO時は、アニメーションGIF生成
if level[FLAGS.verbose] == logging.INFO:
logger.info(
"creating Gif {0}/movie_smoothing.gif, please Wait!".format(
subdir))
imageio.mimsave('{0}/movie_smoothing.gif'.format(subdir),
png_lib,
fps=FLAGS.gif_fps)
logger.info("Done!".format(pngName))
def main(_):
smoothed = read_openpose_json()
logger.info("reading and smoothing done. start feeding 3d-pose-baseline")
plt.figure(2)
smooth_curves_plot = show_anim_curves(smoothed, plt)
pngName = 'png/smooth_plot.png'
smooth_curves_plot.savefig(pngName)
enc_in = np.zeros((1, 64))
enc_in[0] = [0 for i in range(64)]
actions = data_utils.define_actions(FLAGS.action)
SUBJECT_IDS = [1, 5, 6, 7, 8, 9, 11]
rcams = cameras.load_cameras(FLAGS.cameras_path, SUBJECT_IDS)
train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.read_2d_predictions(
actions, FLAGS.data_dir)
train_set_3d, test_set_3d, data_mean_3d, data_std_3d, dim_to_ignore_3d, dim_to_use_3d, train_root_positions, test_root_positions = data_utils.read_3d_data(
actions, FLAGS.data_dir, FLAGS.camera_frame, rcams, FLAGS.predict_14)
device_count = {"GPU": 1}
png_lib = []
with tf.Session(config=tf.ConfigProto(device_count=device_count,
allow_soft_placement=True)) as sess:
#plt.figure(3)
batch_size = 128
model = create_model(sess, actions, batch_size)
for n, (frame, xy) in enumerate(smoothed.items()):
logger.info("calc frame {0}".format(frame))
# map list into np array
joints_array = np.zeros((1, 36))
joints_array[0] = [0 for i in range(36)]
for o in range(len(joints_array[0])):
#feed array with xy array
joints_array[0][o] = xy[o]
_data = joints_array[0]
# mapping all body parts or 3d-pose-baseline format
for i in range(len(order)):
for j in range(2):
# create encoder input
enc_in[0][order[i] * 2 + j] = _data[i * 2 + j]
for j in range(2):
# Hip
enc_in[0][0 * 2 + j] = (enc_in[0][1 * 2 + j] +
enc_in[0][6 * 2 + j]) / 2
# Neck/Nose
enc_in[0][14 * 2 + j] = (enc_in[0][15 * 2 + j] +
enc_in[0][12 * 2 + j]) / 2
# Thorax
enc_in[0][13 * 2 +
j] = 2 * enc_in[0][12 * 2 + j] - enc_in[0][14 * 2 +
j]
# set spine
spine_x = enc_in[0][24]
spine_y = enc_in[0][25]
enc_in = enc_in[:, dim_to_use_2d]
mu = data_mean_2d[dim_to_use_2d]
stddev = data_std_2d[dim_to_use_2d]
enc_in = np.divide((enc_in - mu), stddev)
dp = 1.0
dec_out = np.zeros((1, 48))
dec_out[0] = [0 for i in range(48)]
_, _, poses3d = model.step(sess,
enc_in,
dec_out,
dp,
isTraining=False)
all_poses_3d = []
enc_in = data_utils.unNormalizeData(enc_in, data_mean_2d,
data_std_2d, dim_to_ignore_2d)
poses3d = data_utils.unNormalizeData(poses3d, data_mean_3d,
data_std_3d, dim_to_ignore_3d)
gs1 = gridspec.GridSpec(1, 1)
gs1.update(wspace=-0.00,
hspace=0.05) # set the spacing between axes.
plt.axis('off')
all_poses_3d.append(poses3d)
enc_in, poses3d = map(np.vstack, [enc_in, all_poses_3d])
subplot_idx, exidx = 1, 1
max = 0
min = 10000
for i in range(poses3d.shape[0]):
for j in range(32):
tmp = poses3d[i][j * 3 + 2]
poses3d[i][j * 3 + 2] = poses3d[i][j * 3 + 1]
poses3d[i][j * 3 + 1] = tmp
if poses3d[i][j * 3 + 2] > max:
max = poses3d[i][j * 3 + 2]
if poses3d[i][j * 3 + 2] < min:
min = poses3d[i][j * 3 + 2]
for i in range(poses3d.shape[0]):
for j in range(32):
poses3d[i][j * 3 + 2] = max - poses3d[i][j * 3 + 2] + min
poses3d[i][j * 3] += (spine_x - 630)
poses3d[i][j * 3 + 2] += (500 - spine_y)
# Plot 3d predictions
ax = plt.subplot(gs1[subplot_idx - 1], projection='3d')
ax.view_init(18, -70)
logger.debug(np.min(poses3d))
if np.min(poses3d) < -1000:
poses3d = before_pose
p3d = poses3d
logger.debug(poses3d)
viz.show3Dpose(p3d, ax, lcolor="#9b59b6", rcolor="#2ecc71")
pngName = 'png/test_{0}.png'.format(str(frame))
plt.savefig(pngName)
png_lib.append(imageio.imread(pngName))
before_pose = poses3d
logger.info("creating Gif png/movie_smoothing.gif, please Wait!")
imageio.mimsave('png/movie_smoothing.gif', png_lib, fps=FLAGS.gif_fps)
logger.info("Done!".format(pngName))
def main(_):
done = []
enc_in = np.zeros((1, 64))
enc_in[0] = [0 for i in range(64)]
actions = data_utils.define_actions(FLAGS.action)
SUBJECT_IDS = [1, 5, 6, 7, 8, 9, 11]
rcams = cameras.load_cameras(FLAGS.cameras_path, SUBJECT_IDS)
train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.read_2d_predictions(
actions, FLAGS.data_dir)
train_set_3d, test_set_3d, data_mean_3d, data_std_3d, dim_to_ignore_3d, dim_to_use_3d, train_root_positions, test_root_positions = data_utils.read_3d_data(
actions, FLAGS.data_dir, FLAGS.camera_frame, rcams, FLAGS.predict_14)
device_count = {"GPU": 0}
png_lib = []
with tf.Session(config=tf.ConfigProto(device_count=device_count,
allow_soft_placement=True)) as sess:
#plt.figure(3)
batch_size = 128
model = create_model(sess, actions, batch_size)
while True:
key = cv2.waitKey(1) & 0xFF
#logger.info("start reading data")
# check for other file types
list_of_files = glob.iglob("{0}/*".format(
openpose_output_dir)) # You may use iglob in Python3
latest_file = ""
try:
latest_file = max(list_of_files, key=os.path.getctime)
except ValueError:
#empthy dir
pass
if not latest_file:
continue
try:
_file = file_name = latest_file
print(latest_file)
if not os.path.isfile(_file):
raise Exception("No file found!!, {0}".format(_file))
data = json.load(open(_file))
#take first person
_data = data["people"][0]["pose_keypoints"]
xy = []
#ignore confidence score
for o in range(0, len(_data), 3):
xy.append(_data[o])
xy.append(_data[o + 1])
frame_indx = re.findall("(\d+)", file_name)
frame = int(frame_indx[0])
joints_array = np.zeros((1, 36))
joints_array[0] = [0 for i in range(36)]
for o in range(len(joints_array[0])):
#feed array with xy array
joints_array[0][o] = xy[o]
_data = joints_array[0]
# mapping all body parts or 3d-pose-baseline format
for i in range(len(order)):
for j in range(2):
# create encoder input
enc_in[0][order[i] * 2 + j] = _data[i * 2 + j]
for j in range(2):
# Hip
enc_in[0][0 * 2 + j] = (enc_in[0][1 * 2 + j] +
enc_in[0][6 * 2 + j]) / 2
# Neck/Nose
enc_in[0][14 * 2 + j] = (enc_in[0][15 * 2 + j] +
enc_in[0][12 * 2 + j]) / 2
# Thorax
enc_in[0][
13 * 2 +
j] = 2 * enc_in[0][12 * 2 + j] - enc_in[0][14 * 2 + j]
# set spine
spine_x = enc_in[0][24]
spine_y = enc_in[0][25]
enc_in = enc_in[:, dim_to_use_2d]
mu = data_mean_2d[dim_to_use_2d]
stddev = data_std_2d[dim_to_use_2d]
enc_in = np.divide((enc_in - mu), stddev)
dp = 1.0
dec_out = np.zeros((1, 48))
dec_out[0] = [0 for i in range(48)]
_, _, poses3d = model.step(sess,
enc_in,
dec_out,
dp,
isTraining=False)
all_poses_3d = []
enc_in = data_utils.unNormalizeData(enc_in, data_mean_2d,
data_std_2d,
dim_to_ignore_2d)
poses3d = data_utils.unNormalizeData(poses3d, data_mean_3d,
data_std_3d,
dim_to_ignore_3d)
gs1 = gridspec.GridSpec(1, 1)
gs1.update(wspace=-0.00,
hspace=0.05) # set the spacing between axes.
plt.axis('off')
all_poses_3d.append(poses3d)
enc_in, poses3d = map(np.vstack, [enc_in, all_poses_3d])
subplot_idx, exidx = 1, 1
_max = 0
_min = 10000
for i in range(poses3d.shape[0]):
for j in range(32):
tmp = poses3d[i][j * 3 + 2]
poses3d[i][j * 3 + 2] = poses3d[i][j * 3 + 1]
poses3d[i][j * 3 + 1] = tmp
if poses3d[i][j * 3 + 2] > _max:
_max = poses3d[i][j * 3 + 2]
if poses3d[i][j * 3 + 2] < _min:
_min = poses3d[i][j * 3 + 2]
for i in range(poses3d.shape[0]):
for j in range(32):
poses3d[i][j * 3 +
2] = _max - poses3d[i][j * 3 + 2] + _min
poses3d[i][j * 3] += (spine_x - 630)
poses3d[i][j * 3 + 2] += (500 - spine_y)
# Plot 3d predictions
ax = plt.subplot(gs1[subplot_idx - 1], projection='3d')
ax.view_init(18, -70)
logger.debug(np.min(poses3d))
if np.min(poses3d) < -1000 and frame != 0:
poses3d = before_pose
p3d = poses3d
viz.show3Dpose(p3d, ax, lcolor="#9b59b6", rcolor="#2ecc71")
before_pose = poses3d
pngName = 'png/test_{0}.png'.format(str(frame))
plt.savefig(pngName)
#plt.show()
img = cv2.imread(pngName, 0)
rect_cpy = img.copy()
cv2.imshow('3d-pose-baseline', rect_cpy)
done.append(file_name)
if key == ord('q'):
break
except Exception as e:
print(e)
sess.close()
def main(_):
global framenum
#clear out all old frames
os.system("rm png/*")
#set done to empty array, it will hold the json files from openpose that we've already processed
done = []
#initialize input tensor to 1x64 array of zeroes [[0. 0. 0. ...]]
#this is list of numpy vectors to feed as encoder inputs (32 2d coordinates)
enc_in = np.zeros((1, 64))
enc_in[0] = [0 for i in range(64)]
#actions to run on, default is all
actions = data_utils.define_actions(FLAGS.action)
#the list of Human3.6m subjects to look at
SUBJECT_IDS = [1, 5, 6, 7, 8, 9, 11]
#load camera parameters from the h36m dataset
rcams = cameras2.load_cameras(FLAGS.cameras_path, SUBJECT_IDS)
#loads 2d data from precomputed Stacked Hourglass detections
train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.read_2d_predictions(
actions, FLAGS.data_dir)
#loads 3d poses, zero-centres and normalizes them
train_set_3d, test_set_3d, data_mean_3d, data_std_3d, dim_to_ignore_3d, dim_to_use_3d, train_root_positions, test_root_positions = data_utils.read_3d_data(
actions, FLAGS.data_dir, FLAGS.camera_frame, rcams, FLAGS.predict_14)
device_count = {"GPU": 0}
png_lib = []
#run a tensorflow inference session
with tf.Session(config=tf.ConfigProto(device_count=device_count,
allow_soft_placement=True)) as sess:
#plt.figure(3)
#load pre-trained model
batch_size = 128
model = create_model(sess, actions, batch_size)
#infinitely show 3d pose visualization
while True:
#wait for key to be pressed
key = cv2.waitKey(1) & 0xFF
_, frame = cv2.VideoCapture(
0).read() #ignore the other returned value
#resize and rotate the incoming image frame
frame, W, H = resize_img(frame)
frame = cv2.rotate(frame, cv2.ROTATE_90_COUNTERCLOCKWISE)
start = time.time()
#run posenet inference on the frame
joints_2d = estimate_pose(frame)
#throw out confidence score and flatten
_data = joints_2d[..., :2].flatten()
#open pop-up and draw the keypoints found
img2D = draw_2Dimg(frame, joints_2d, 1)
#fake the thorax point by finding midpt between left and right shoulder
lt_should_x = _data[10]
lt_should_y = _data[11]
rt_should_x = _data[12]
rt_should_y = _data[13]
thorax = midpoint(lt_should_x, lt_should_y, rt_should_x,
rt_should_y)
#print("testing thorax pt at ", thorax)
#insert thorax into data where it should be, at index 1
_data = np.insert(_data, 2, thorax[0])
_data = np.insert(_data, 3, thorax[1])
#print("new _data is ", _data)
_data = np.around(_data)
#set xy to the array of 2d joint data
xy = _data
#create new 1x36 array of zeroes, which will store the 18 2d keypoints
joints_array = np.zeros((1, 36))
joints_array[0] = [0 for i in range(36)]
#index into our data array
index = 0
#iterates 18 times
for o in range(int(len(joints_array[0]) / 2)):
#feed array with xy array (the 18 keypoints), but switch ordering: posenet to openpose
for j in range(2):
#print("o is", o, "j is", j, "index is ", index)
index_into_posenet_data = order_pnet_to_openpose[o] * 2 + j
#print("putting posenet[", index_into_posenet_data, "], value ", xy[index_into_posenet_data], " , into joints_array[0][", index, "]")
joints_array[0][index] = xy[index_into_posenet_data]
index += 1
#set _data to the array containing the 36 coordinates of the 2d keypts
_data = joints_array[0]
#print("_data is ", _data)
#mapping all body parts for 3d-pose-baseline format (32 2d coordinates)
for i in range(len(order)): #iterates 14 times
#select which coordinateof this point: x or y
for j in range(2):
#create encoder input, switching around the order of the joint points
enc_in[0][order[i] * 2 + j] = _data[i * 2 + j]
#now enc_in contains 14 points (28 total coordinates)
#at this pt enc_in should be array of 64 vals
for j in range(2):
#place hip at index 0
enc_in[0][0 * 2 + j] = (enc_in[0][1 * 2 + j] +
enc_in[0][6 * 2 + j]) / 2
#place neck/nose at index 14
enc_in[0][14 * 2 + j] = (enc_in[0][15 * 2 + j] +
enc_in[0][12 * 2 + j]) / 2
#place thorax at index 13
enc_in[0][13 * 2 +
j] = 2 * enc_in[0][12 * 2 + j] - enc_in[0][14 * 2 +
j]
#set spine found by openpose
spine_x = enc_in[0][24]
spine_y = enc_in[0][25]
#dim_to_use_2d is always [0 1 2 3 4 5 6 7 12 13 14 15 16 17 24 25 26 27 30 31 34 35 36 37 38 39 50 51 52 53 54 55]
#take 32 entries of enc_in
enc_in = enc_in[:, dim_to_use_2d]
#find mean of 2d data
mu = data_mean_2d[dim_to_use_2d]
#find stdev of 2d data
stddev = data_std_2d[dim_to_use_2d]
#subtract mean and divide std for all
enc_in = np.divide((enc_in - mu), stddev)
#dropout keep probability
dp = 1.0
#output tensor, initialize it to zeroes. We'll get 16 joints with 3d coordinates
#this is list of numpy vectors that are the expected decoder outputs
dec_out = np.zeros((1, 48))
dec_out[0] = [0 for i in range(48)]
#get the 3d poses by running the 3d-pose-baseline inference. Model operates on 32 points
_, _, poses3d = model.step(sess,
enc_in,
dec_out,
dp,
isTraining=False)
#poses3d comes back as a 1x96 array (I guess its 32 points)
end = time.time()
#print("ELAPSED: ", end-start)
#hold our 3d poses while we're doing some post-processing
all_poses_3d = []
#un-normalize the input and output data using the means and stdevs
enc_in = data_utils.unNormalizeData(enc_in, data_mean_2d,
data_std_2d, dim_to_ignore_2d)
poses3d = data_utils.unNormalizeData(poses3d, data_mean_3d,
data_std_3d, dim_to_ignore_3d)
#create a grid for drawing
gs1 = gridspec.GridSpec(1, 1)
#set spacing between axes
gs1.update(wspace=-0.00, hspace=0.05)
plt.axis('off')
#fill all_poses_3d with the 3d poses predicted by the model step fxn
all_poses_3d.append(poses3d)
#vstack stacks arrays in sequence vertically (row wise)
#this doesn't do anything in this case, as far as I can tell
enc_in, poses3d = map(np.vstack, [enc_in, all_poses_3d])
subplot_idx, exidx = 1, 1
_max = 0
_min = 10000
#iterates once
for i in range(poses3d.shape[0]):
#iterate over all 32 points in poses3d
for j in range(32):
#save the last coordinate of this point into tmp
tmp = poses3d[i][j * 3 + 2]
#swap the second and third coordinates of this pt
poses3d[i][j * 3 + 2] = poses3d[i][j * 3 + 1]
poses3d[i][j * 3 + 1] = tmp
#keep track of max of last coordinate
if poses3d[i][j * 3 + 2] > _max:
_max = poses3d[i][j * 3 + 2]
if poses3d[i][j * 3 + 2] < _min:
_min = poses3d[i][j * 3 + 2]
#iterates once
for i in range(poses3d.shape[0]):
#iterate over all 32 points in poses3d (2nd and 3rd coords have all been swapped at this pt)
for j in range(32):
#change the third coord of this pt, subtracting it from sum of max and min third coord to get new value
poses3d[i][j * 3 + 2] = _max - poses3d[i][j * 3 + 2] + _min
#modify first coord of this pt by adding the x coord of the spine found by 2d model
poses3d[i][j * 3] += (spine_x - 630)
#modify third coord of this pt by adding 500 minus y coord of spine found by 2d model
poses3d[i][j * 3 + 2] += (500 - spine_y)
#Plot 3d predictions
ax = plt.subplot(gs1[subplot_idx - 1], projection='3d')
ax.view_init(18, -70)
logger.debug(np.min(poses3d))
#TODO: if something happened with the data, reuse data from last frame (before_pose)
p3d = poses3d
#plot the 3d skeleton
viz.show3Dpose(p3d, ax, lcolor="#9b59b6", rcolor="#2ecc71")
#keep track of this poses3d in case we need to reuse it for next frame
before_pose = poses3d
#save this frame as a png in the ./png/ folder
pngName = 'png/test_{0}.png'.format(str(framenum))
#print("pngName is ", pngName)
plt.savefig(pngName)
#plt.show()
#read this frame which was just saved as png
img = cv2.imread(pngName, 0)
rect_cpy = img.copy()
#show this frame
cv2.imshow('3d-pose-baseline', rect_cpy)
framenum += 1
#quit if q is pressed
if key == ord('q'):
break
sess.close()
def main(_):
# 出力用日付
now_str = "{0:%Y%m%d_%H%M%S}".format(datetime.datetime.now())
logger.debug("FLAGS.person_idx={0}".format(FLAGS.person_idx))
# ディレクトリ構成が変わったので、JSON出力と同階層に出力(2/9)
if FLAGS.output is None:
subdir = openpose_output_dir
else:
subdir = FLAGS.output
os.makedirs(subdir, exist_ok=True)
frame3d_dir = "{0}/frame3d".format(subdir)
if os.path.exists(frame3d_dir):
# 既にディレクトリがある場合、一旦削除
shutil.rmtree(frame3d_dir)
os.makedirs(frame3d_dir)
#関節位置情報ファイル
posf = open(subdir + '/pos.txt', 'w')
#正規化済みOpenpose位置情報ファイル
smoothedf = open(subdir + '/smoothed.txt', 'w')
#開始フレームインデックスファイル
start_frame_f = open(subdir + '/start_frame.txt', 'w')
idx = FLAGS.person_idx - 1
start_frame_index, smoothed = openpose_utils.read_openpose_json(
"{0}/json".format(openpose_output_dir), idx, FLAGS.verbose == 3)
# 開始フレームインデックスを保存
start_frame_f.write(str(start_frame_index))
start_frame_f.close()
logger.info("reading and smoothing done. start feeding 3d-pose-baseline")
logger.debug(smoothed)
plt.figure(2)
smooth_curves_plot = show_anim_curves(smoothed, plt)
pngName = subdir + '/smooth_plot.png'
smooth_curves_plot.savefig(pngName)
enc_in = np.zeros((1, 64))
enc_in[0] = [0 for i in range(64)]
actions = data_utils.define_actions(FLAGS.action)
SUBJECT_IDS = [1, 5, 6, 7, 8, 9, 11]
rcams = cameras.load_cameras(FLAGS.cameras_path, SUBJECT_IDS)
train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.read_2d_predictions(
actions, FLAGS.data_dir)
train_set_3d, test_set_3d, data_mean_3d, data_std_3d, dim_to_ignore_3d, dim_to_use_3d, train_root_positions, test_root_positions = data_utils.read_3d_data(
actions, FLAGS.data_dir, FLAGS.camera_frame, rcams, FLAGS.predict_14)
# before_pose = None
device_count = {"GPU": 1}
png_lib = []
with tf.Session(config=tf.ConfigProto(device_count=device_count,
allow_soft_placement=True)) as sess:
#plt.figure(3)
batch_size = 128
model = create_model(sess, actions, batch_size)
# 入力画像のスケール調整のため、NeckからHipまでの距離を測定
length_neck2hip_mean = get_length_neck2hip_mean(smoothed)
# 2D、3D結果の保存用リスト
poses3d_list = []
poses2d_list = []
# 2dと3dのスケール比率計算のためのリスト
length_2d_list = []
length_3d_list = []
for n, (frame, xy) in enumerate(smoothed.items()):
if frame % 200 == 0:
logger.info("calc idx {0}, frame {1}".format(idx, frame))
#if frame % 300 == 0:
# print(frame)
# map list into np array
joints_array = np.zeros((1, 36))
joints_array[0] = [0 for i in range(36)]
for o in range(len(joints_array[0])):
#feed array with xy array
joints_array[0][o] = xy[o]
_data = joints_array[0]
smoothedf.write(' '.join(map(str, _data)))
smoothedf.write("\n")
# mapping all body parts or 3d-pose-baseline format
for i in range(len(order)):
for j in range(2):
# create encoder input
enc_in[0][order[i] * 2 + j] = _data[i * 2 + j]
for j in range(2):
# Hip
enc_in[0][0 * 2 + j] = (enc_in[0][1 * 2 + j] +
enc_in[0][6 * 2 + j]) / 2
# Thorax
# 3dPoseBaselineのThoraxの位置は、OpenPoseのNeckの位置より少し上のため調整する
enc_in[0][13 * 2 +
j] = 1.1 * enc_in[0][13 * 2 +
j] - 0.1 * enc_in[0][0 * 2 + j]
# Neck/Nose
enc_in[0][14 * 2 + j] = (enc_in[0][15 * 2 + j] +
enc_in[0][13 * 2 + j]) / 2
# Spine
enc_in[0][12 * 2 + j] = (enc_in[0][0 * 2 + j] +
enc_in[0][13 * 2 + j]) / 2
# set spine
# spine_x = enc_in[0][24]
# spine_y = enc_in[0][25]
# logger.debug("enc_in - 1")
# logger.debug(enc_in)
poses2d = enc_in
# 入力データの拡大
# neckからHipまでが110ピクセル程度になるように入力を拡大する
# (教師データとスケールが大きく異なると精度が落ちるため)
input_scaling_factor = 110 / length_neck2hip_mean
enc_in = enc_in * input_scaling_factor
enc_in = enc_in[:, dim_to_use_2d]
mu = data_mean_2d[dim_to_use_2d]
stddev = data_std_2d[dim_to_use_2d]
enc_in = np.divide((enc_in - mu), stddev)
dp = 1.0
dec_out = np.zeros((1, 48))
dec_out[0] = [0 for i in range(48)]
_, _, poses3d = model.step(sess,
enc_in,
dec_out,
dp,
isTraining=False)
all_poses_3d = []
enc_in = data_utils.unNormalizeData(enc_in, data_mean_2d,
data_std_2d, dim_to_ignore_2d)
poses3d = data_utils.unNormalizeData(poses3d, data_mean_3d,
data_std_3d, dim_to_ignore_3d)
gs1 = gridspec.GridSpec(1, 1)
gs1.update(wspace=-0.00,
hspace=0.05) # set the spacing between axes.
plt.axis('off')
all_poses_3d.append(poses3d)
enc_in, poses3d = map(np.vstack, [enc_in, all_poses_3d])
subplot_idx, exidx = 1, 1
poses3d_list.append(poses3d[0])
poses2d_list.append(poses2d[0])
length_2d_list.append(sum_length_xy(poses2d[0], 2))
length_3d_list.append(sum_length_xy(poses3d[0], 3))
# OpenPose出力の(x, y)とBaseline出力のzから、3次元の位置を計算する
# OpenPose出力値とBaseline出力値のスケール比率
# 骨格の長さの合計の比較することで、比率を推定
# 前後の91フレームで移動平均をとることで、結果を安定化する
move_ave_length_2d = calc_move_average(length_2d_list, 91)
move_ave_length_3d = calc_move_average(length_3d_list, 91)
move_ave_length_2d[move_ave_length_2d == 0] = 1 # error防止
xy_scale = move_ave_length_3d / move_ave_length_2d
# 以下の４つは仮の値で計算。多少違っていても、精度に影響はないと思う
center_2d_x, center_2d_y = camera_center(
openpose_output_dir) #動画の中心座標（動画の解像度の半分）
logger.info("center_2d_x {0}".format(center_2d_x))
z_distance = 4000 # カメラから体までの距離(mm) 遠近の影響計算で使用
camera_incline = 0 # カメラの水平方向に対する下への傾き（度）
teacher_camera_incline = 13 # 教師データ(Human3.6M)のカメラの傾き（下向きに平均13度）
for frame, (poses3d,
poses2d) in enumerate(zip(poses3d_list, poses2d_list)):
# 誤差を減らすため、OpenPose出力の(x, y)と3dPoseBaseline出力のzから、3次元の位置を計算する
poses3d_op_xy = np.zeros(96)
for i in [0, 1, 2, 3, 6, 7, 8, 13, 15, 17, 18, 19, 25, 26, 27]:
# Hipとの差分
dy = poses3d[i * 3 + 1] - poses3d[0 * 3 + 1]
dz = poses3d[i * 3 + 2] - poses3d[0 * 3 + 2]
# 教師データのカメラ傾きを補正
dz = dz - dy * math.tan(
math.radians(teacher_camera_incline - camera_incline))
# 遠近によるx,yの拡大率
z_ratio = (z_distance + dz) / z_distance
# x, yはOpenposeの値から計算
poses3d_op_xy[i *
3] = (poses2d[i * 2] -
center_2d_x) * xy_scale[frame] * z_ratio
poses3d_op_xy[i * 3 +
1] = (poses2d[i * 2 + 1] -
center_2d_y) * xy_scale[frame] * z_ratio
# zはBaselineの値から計算
poses3d_op_xy[i * 3 + 2] = dz
# 12(Spine)、14(Neck/Nose)、15(Head)はOpenPoseの出力にないため、baseline(poses3d)から計算する
for i in [12, 14, 15]:
# 13(Thorax)は認識されることが多いため基準とする
# 差分
dx = poses3d[i * 3] - poses3d[13 * 3]
dy = poses3d[i * 3 + 1] - poses3d[13 * 3 + 1]
dz = poses3d[i * 3 + 2] - poses3d[13 * 3 + 2]
# 教師データのカメラ傾きを補正
dz = dz - dy * math.tan(
math.radians(teacher_camera_incline - camera_incline))
# 13(Thorax)からの差分でx, y ,zを求める
poses3d_op_xy[i * 3] = poses3d_op_xy[13 * 3] + dx
poses3d_op_xy[i * 3 + 1] = poses3d_op_xy[13 * 3 + 1] + dy
poses3d_op_xy[i * 3 + 2] = poses3d_op_xy[13 * 3 + 2] + dz
# MMD上で少し顎を引くための処理
poses3d_op_xy[15 * 3] += 0.5 * (poses3d_op_xy[14 * 3] -
poses3d_op_xy[13 * 3])
poses3d_op_xy[15 * 3 + 1] += 0.5 * (poses3d_op_xy[14 * 3 + 1] -
poses3d_op_xy[13 * 3 + 1])
poses3d_op_xy[15 * 3 + 2] += 0.5 * (poses3d_op_xy[14 * 3 + 2] -
poses3d_op_xy[13 * 3 + 2])
poses3d_list[frame] = poses3d_op_xy
logger.info("calc ground y")
# 最も高さが低い足の部位のYを取得(この座標系ではY値が大きい方が低い)
foot_joint_no = [1, 2, 3, 6, 7, 8]
max_pos = []
for frame, poses3d in enumerate(poses3d_list):
max_pos.append(np.max([poses3d[i * 3 + 1] for i in foot_joint_no]))
# 地面についている部位の位置（通常は足首）をY軸の0になるように移動する
for frame, poses3d in enumerate(poses3d_list):
# 120フレーム分の位置を取得
max_pos_slice = max_pos[int(np.max([0, frame - 60])):frame + 60]
# 半分以上のフレームでは着地していると仮定し、メディアンを着地時の足の位置とする
ankle_pos = np.median(max_pos_slice)
poses3d_ground = np.zeros(96)
for i in range(len(data_utils.H36M_NAMES)):
poses3d_ground[i * 3] = poses3d[i * 3]
poses3d_ground[i * 3 + 1] = poses3d[i * 3 + 1] - ankle_pos
poses3d_ground[i * 3 + 2] = poses3d[i * 3 + 2]
poses3d_list[frame] = poses3d_ground
for frame, (poses3d,
poses2d) in enumerate(zip(poses3d_list, poses2d_list)):
if frame % 200 == 0:
logger.info("output frame {}".format(frame))
# max = 0
# min = 10000
# logger.debug("enc_in - 2")
# logger.debug(enc_in)
for j in range(32):
tmp = poses3d[j * 3 + 2]
poses3d[j * 3 + 2] = -poses3d[j * 3 + 1]
poses3d[j * 3 + 1] = tmp
# if poses3d[i][j * 3 + 2] > max:
# max = poses3d[i][j * 3 + 2]
# if poses3d[i][j * 3 + 2] < min:
# min = poses3d[i][j * 3 + 2]
# for i in range(poses3d.shape[0]):
# for j in range(32):
# poses3d[i][j * 3 + 2] = max - poses3d[i][j * 3 + 2] + min
# poses3d[i][j * 3] += (spine_x - 630)
# poses3d[i][j * 3 + 2] += (500 - spine_y)
# Plot 3d predictions
ax = plt.subplot(gs1[subplot_idx - 1], projection='3d')
ax.view_init(18, 280)
# logger.debug(np.min(poses3d))
# if np.min(poses3d) < -1000 and before_pose is not None:
# poses3d = before_pose
p3d = poses3d
# logger.debug("poses3d")
# logger.debug(poses3d)
if frame == 0:
first_xyz = [0, 0, 0]
first_xyz[0], first_xyz[1], first_xyz[2] = p3d[0], p3d[1], p3d[
2]
if level[FLAGS.verbose] <= logging.INFO:
viz.show3Dpose(p3d,
ax,
lcolor="#9b59b6",
rcolor="#2ecc71",
add_labels=True,
root_xyz=first_xyz)
# 各フレームの単一視点からのはINFO時のみ
pngName = frame3d_dir + '/tmp_{0:012d}.png'.format(frame)
plt.savefig(pngName)
png_lib.append(imageio.imread(pngName))
# before_pose = poses3d
# 各フレームの角度別出力はデバッグ時のみ
if level[FLAGS.verbose] == logging.DEBUG:
for azim in [0, 45, 90, 135, 180, 225, 270, 315, 360]:
ax2 = plt.subplot(gs1[subplot_idx - 1], projection='3d')
ax2.view_init(18, azim)
viz.show3Dpose(p3d,
ax2,
lcolor="#FF0000",
rcolor="#0000FF",
add_labels=True,
root_xyz=first_xyz)
pngName2 = frame3d_dir + '/tmp_{0:012d}_{1:03d}.png'.format(
frame, azim)
plt.savefig(pngName2)
#関節位置情報の出力
write_pos_data(poses3d, ax, posf)
posf.close()
smoothedf.close()
# INFO時は、アニメーションGIF生成
if level[FLAGS.verbose] <= logging.INFO:
logger.info(
"creating Gif {0}/movie_smoothing.gif, please Wait!".format(
subdir))
imageio.mimsave('{0}/movie_smoothing.gif'.format(subdir),
png_lib,
fps=FLAGS.gif_fps)
logger.info("Done!".format(pngName))
def main(_):
done = []
enc_in = np.zeros((1, 64))
enc_in[0] = [0 for i in range(64)]
actions = data_utils.define_actions(FLAGS.action)
SUBJECT_IDS = [1, 5, 6, 7, 8, 9, 11]
rcams = cameras.load_cameras(FLAGS.cameras_path, SUBJECT_IDS)
train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.read_2d_predictions(
actions, FLAGS.data_dir)
train_set_3d, test_set_3d, data_mean_3d, data_std_3d, dim_to_ignore_3d, dim_to_use_3d, train_root_positions, test_root_positions = data_utils.read_3d_data(
actions, FLAGS.data_dir, FLAGS.camera_frame, rcams, FLAGS.predict_14)
device_count = {"GPU": 0}
png_lib = []
with tf.Session(config=tf.ConfigProto(device_count=device_count,
allow_soft_placement=True)) as sess:
#plt.figure(3)
batch_size = 128
model = create_model(sess, actions, batch_size)
rows = 0
filename = "Realtimedata.xlsx"
workbook = xlsxwriter.Workbook(filename)
worksheet = workbook.add_worksheet()
while True:
key = cv2.waitKey(1) & 0xFF
#logger.info("start reading data")
# check for other file types
list_of_files = glob.iglob("{0}/*".format(
openpose_output_dir)) # You may use iglob in Python3
latest_file = ""
try:
latest_file = max(list_of_files, key=os.path.getctime)
except ValueError:
#empthy dir
pass
if not latest_file:
continue
try:
_file = file_name = latest_file
print(latest_file)
if not os.path.isfile(_file):
raise Exception("No file found!!, {0}".format(_file))
data = json.load(open(_file))
#take first person
_data = data["people"][0]["pose_keypoints_2d"]
xy = []
#ignore confidence score
"""for o in range(0,len(_data),3):
xy.append(_data[o])
xy.append(_data[o+1])"""
if len(_data) >= 53:
#openpose incl. confidence score
#ignore confidence score
for o in range(0, len(_data), 3):
xy.append(_data[o])
xy.append(_data[o + 1])
else:
#tf-pose-estimation
xy = _data
frame_indx = re.findall("(\d+)", file_name)
frame = int(frame_indx[0])
joints_array = np.zeros((1, 36))
joints_array[0] = [0 for i in range(36)]
for o in range(len(joints_array[0])):
#feed array with xy array
joints_array[0][o] = xy[o]
_data = joints_array[0]
# mapping all body parts or 3d pose offline format
for i in range(len(order)):
for j in range(2):
# create encoder input
enc_in[0][order[i] * 2 + j] = _data[i * 2 + j]
for j in range(2):
# Hip
enc_in[0][0 * 2 + j] = (enc_in[0][1 * 2 + j] +
enc_in[0][6 * 2 + j]) / 2
# Neck/Nose
enc_in[0][14 * 2 + j] = (enc_in[0][15 * 2 + j] +
enc_in[0][12 * 2 + j]) / 2
# Thorax
enc_in[0][
13 * 2 +
j] = 2 * enc_in[0][12 * 2 + j] - enc_in[0][14 * 2 + j]
# set spine
spine_x = enc_in[0][24]
spine_y = enc_in[0][25]
enc_in = enc_in[:, dim_to_use_2d]
mu = data_mean_2d[dim_to_use_2d]
stddev = data_std_2d[dim_to_use_2d]
enc_in = np.divide((enc_in - mu), stddev)
dp = 1.0
dec_out = np.zeros((1, 48))
dec_out[0] = [0 for i in range(48)]
_, _, poses3d = model.step(sess,
enc_in,
dec_out,
dp,
isTraining=False)
all_poses_3d = []
enc_in = data_utils.unNormalizeData(enc_in, data_mean_2d,
data_std_2d,
dim_to_ignore_2d)
poses3d = data_utils.unNormalizeData(poses3d, data_mean_3d,
data_std_3d,
dim_to_ignore_3d)
gs1 = gridspec.GridSpec(1, 1)
gs1.update(wspace=-0.00,
hspace=0.05) # set the spacing between axes.
plt.axis('off')
all_poses_3d.append(poses3d)
enc_in, poses3d = map(np.vstack, [enc_in, all_poses_3d])
subplot_idx, exidx = 1, 1
_max = 0
_min = 10000
for i in range(poses3d.shape[0]):
for j in range(32):
tmp = poses3d[i][j * 3 + 2]
poses3d[i][j * 3 + 2] = poses3d[i][j * 3 + 1]
poses3d[i][j * 3 + 1] = tmp
if poses3d[i][j * 3 + 2] > _max:
_max = poses3d[i][j * 3 + 2]
if poses3d[i][j * 3 + 2] < _min:
_min = poses3d[i][j * 3 + 2]
for i in range(poses3d.shape[0]):
for j in range(32):
poses3d[i][j * 3 +
2] = _max - poses3d[i][j * 3 + 2] + _min
poses3d[i][j * 3] += (spine_x - 630)
poses3d[i][j * 3 + 2] += (500 - spine_y)
for val in min_vex:
# f.write(str(val) + ' ' + str(p_vex[i]) + '');
# gait_list1.append({'IX': "%i" % val[0],
# 'IY': "%i" % val[1],
# 'Ix': "%i" % p_vex[i][0],
# 'Iy': "%i" % p_vex[i][1],
# 'Iz': "%i" % p_vex[i][2],
# })
gait_list1.append(val[0])
gait_list1.append(val[1])
gait_list1.append(p_vex[i][0])
gait_list1.append(p_vex[i][1])
gait_list1.append(p_vex[i][2])
points.append(
" %f %f %f %d %d %d 0\n" %
(p_vex[i][0], p_vex[i][1], p_vex[i][2], 0, 255, 0))
x.append(p_vex[i][0])
y.append(p_vex[i][1])
z.append(p_vex[i][2])
i = i + 1
# Plot 3d predictions
ax = plt.subplot(gs1[subplot_idx - 1], projection='3d')
ax.view_init(18, -70)
logger.debug(np.min(poses3d))
if np.min(poses3d) < -1000 and frame != 0:
poses3d = before_pose
p3d = poses3d
'''gait_list1 = []
#enter file path below
with open('key_joint_info.csv', 'w', newline='') as myfile:
gait_list2.append(gait_list1)
data1 = pd.DataFrame(gait_list2)
wr = csv.writer(myfile, dialect = 'key_joint_info.csv' )
wr.writerow(p3d)
wb.save(key_joint_info.csv)'''
viz.show3Dpose(p3d, ax, lcolor="#9b59b6", rcolor="#2ecc71")
col = 0
for i in p3d[0]:
worksheet.write(rows, col, i)
col += 1
#.append(i)
rows += 1
before_pose = poses3d
pngName = '{}_keypoints.png'.format(str(frame))
plt.savefig(pngName)
#plt.show()
img = cv2.imread(pngName, 0)
rect_cpy = img.copy()
cv2.imshow('3d-pose-realtime', rect_cpy)
done.append(file_name)
if key == ord('q'):
break
except Exception as e:
print(e)
sess.close()
def callback(data):
if data.data == 'hello':
print(data.data)
else:
global FLAGS
global enc_in
global actions
global order
global SUBJECT_IDS
global rcams
global train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d
global train_set_3d, test_set_3d, data_mean_3d, data_std_3d, dim_to_ignore_3d, dim_to_use_3d, train_root_positions, test_root_positions
global device_count
input_data = np.array(eval(data.data))
print('input data shape:{0}'.format(input_data.shape))
tf.reset_default_graph()
with tf.Session(config=tf.ConfigProto(
device_count=device_count, allow_soft_placement=True)) as sess:
batch_size = 128
model = create_model(sess, actions, batch_size)
# map list into np array
joints_array = np.zeros((1, 36))
joints_array[0] = [0 for i in range(36)]
for o in range(len(joints_array[0])):
#feed array with xy array
joints_array[0] = input_data.flatten()
_data = joints_array[0]
# mapping all body parts or 3d-pose-baseline format
for i in range(len(order)):
for j in range(2):
# create encoder input
enc_in[0][order[i] * 2 + j] = _data[i * 2 + j]
for j in range(2):
# Hip
enc_in[0][0 * 2 + j] = (enc_in[0][1 * 2 + j] +
enc_in[0][6 * 2 + j]) / 2
# Neck/Nose
enc_in[0][14 * 2 + j] = (enc_in[0][15 * 2 + j] +
enc_in[0][12 * 2 + j]) / 2
# Thorax
enc_in[0][13 * 2 +
j] = 2 * enc_in[0][12 * 2 + j] - enc_in[0][14 * 2 +
j]
# set spine
spine_x = enc_in[0][24]
spine_y = enc_in[0][25]
enc_in = enc_in[:, dim_to_use_2d]
mu = data_mean_2d[dim_to_use_2d]
stddev = data_std_2d[dim_to_use_2d]
enc_in = np.divide((enc_in - mu), stddev)
dp = 1.0
dec_out = np.zeros((1, 48))
dec_out[0] = [0 for i in range(48)]
_, _, poses3d = model.step(sess,
enc_in,
dec_out,
dp,
isTraining=False)
enc_in = data_utils.unNormalizeData(enc_in, data_mean_2d,
data_std_2d, dim_to_ignore_2d)
poses3d = data_utils.unNormalizeData(poses3d, data_mean_3d,
data_std_3d, dim_to_ignore_3d)
print('output data shape:{0}'.format(poses3d.shape))
def main(_):
smoothed = read_openpose_json()
logger.info("reading and smoothing done. start feeding 3d-pose-baseline")
plt.figure(2)
smooth_curves_plot = show_anim_curves(smoothed, plt)
pngName = FLAGS.output_dirname + '/supplemental' + '/smooth_plot.png'
smooth_curves_plot.savefig(pngName)
if FLAGS.interpolation:
framerange = len(smoothed.keys())
joint_rows = 36
array = np.concatenate(list(smoothed.values()))
array_reshaped = np.reshape(array, (framerange, joint_rows))
multiplier = 0.1
multiplier_inv = 1 / multiplier
out_array = np.array([])
for row in range(joint_rows):
x = []
for frame in range(framerange):
x.append(array_reshaped[frame, row])
frame = range(framerange)
frame_resampled = np.arange(0, framerange, multiplier)
spl = UnivariateSpline(frame, x, k=3)
min_x, max_x = min(x), max(x)
smooth_fac = max_x - min_x
smooth_fac = smooth_fac * 125
spl.set_smoothing_factor(float(smooth_fac))
xnew = spl(frame_resampled)
out_array = np.append(out_array, xnew)
logger.info("done interpolating")
a = np.array([])
for frame in range(int(framerange * multiplier_inv)):
jnt_array = []
#print(frame)
for jnt in range(joint_rows):
jnt_array.append(
out_array[jnt * int(framerange * multiplier_inv) + frame])
a = np.append(a, jnt_array)
a = np.reshape(a, (int(framerange * multiplier_inv), joint_rows))
out_array = a
interpolate_smoothed = {}
for frame in range(int(framerange * multiplier_inv)):
interpolate_smoothed[frame] = list(out_array[frame])
plt.figure(3)
smoothed = interpolate_smoothed
interpolate_curves_plot = show_anim_curves(smoothed, plt)
pngName = FLAGS.output_dirname + '/supplemental' + '/interpolate_plot.png'
interpolate_curves_plot.savefig(pngName)
enc_in = np.zeros((1, 64))
enc_in[0] = [0 for i in range(64)]
actions = data_utils.define_actions(FLAGS.action)
SUBJECT_IDS = [1, 5, 6, 7, 8, 9, 11]
rcams = cameras.load_cameras(FLAGS.cameras_path, SUBJECT_IDS)
train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.read_2d_predictions(
actions, FLAGS.data_dir)
train_set_3d, test_set_3d, data_mean_3d, data_std_3d, dim_to_ignore_3d, dim_to_use_3d, train_root_positions, test_root_positions = data_utils.read_3d_data(
actions, FLAGS.data_dir, FLAGS.camera_frame, rcams, FLAGS.predict_14)
device_count = {"GPU": 1}
png_lib = []
with tf.Session(config=tf.ConfigProto(device_count=device_count,
allow_soft_placement=True)) as sess:
plt.figure(4)
batch_size = 128
model = create_model(sess, actions, batch_size)
for n, (frame, xy) in enumerate(smoothed.items()):
logger.info("calc frame {0}".format(frame))
# map list into np array
joints_array = np.zeros((1, 36))
joints_array[0] = [0 for i in range(36)]
for o in range(len(joints_array[0])):
#feed array with xy array
joints_array[0][o] = xy[o]
_data = joints_array[0]
# mapping all body parts or 3d-pose-baseline format
for i in range(len(order)):
for j in range(2):
# create encoder input
enc_in[0][order[i] * 2 + j] = _data[i * 2 + j]
for j in range(2):
# Hip
enc_in[0][0 * 2 + j] = (enc_in[0][1 * 2 + j] +
enc_in[0][6 * 2 + j]) / 2
# Neck/Nose
enc_in[0][14 * 2 + j] = (enc_in[0][15 * 2 + j] +
enc_in[0][12 * 2 + j]) / 2
# Thorax
enc_in[0][13 * 2 +
j] = 2 * enc_in[0][12 * 2 + j] - enc_in[0][14 * 2 +
j]
# set spine
spine_x = enc_in[0][24]
spine_y = enc_in[0][25]
enc_in = enc_in[:, dim_to_use_2d]
mu = data_mean_2d[dim_to_use_2d]
stddev = data_std_2d[dim_to_use_2d]
enc_in = np.divide((enc_in - mu), stddev)
dp = 1.0
dec_out = np.zeros((1, 48))
dec_out[0] = [0 for i in range(48)]
_, _, poses3d = model.step(sess,
enc_in,
dec_out,
dp,
isTraining=False)
all_poses_3d = []
enc_in = data_utils.unNormalizeData(enc_in, data_mean_2d,
data_std_2d, dim_to_ignore_2d)
poses3d = data_utils.unNormalizeData(poses3d, data_mean_3d,
data_std_3d, dim_to_ignore_3d)
gs1 = gridspec.GridSpec(1, 1)
gs1.update(wspace=-0.00,
hspace=0.05) # set the spacing between axes.
plt.axis('off')
all_poses_3d.append(poses3d)
enc_in, poses3d = map(np.vstack, [enc_in, all_poses_3d])
subplot_idx, exidx = 1, 1
_max = 0
_min = 10000
for i in range(poses3d.shape[0]):
for j in range(32):
tmp = poses3d[i][j * 3 + 2]
poses3d[i][j * 3 + 2] = poses3d[i][j * 3 + 1]
poses3d[i][j * 3 + 1] = tmp
if poses3d[i][j * 3 + 2] > _max:
_max = poses3d[i][j * 3 + 2]
if poses3d[i][j * 3 + 2] < _min:
_min = poses3d[i][j * 3 + 2]
for i in range(poses3d.shape[0]):
for j in range(32):
poses3d[i][j * 3 + 2] = _max - poses3d[i][j * 3 + 2] + _min
poses3d[i][j * 3] += (spine_x - 630)
poses3d[i][j * 3 + 2] += (500 - spine_y)
# Plot 3d predictions
ax = plt.subplot(gs1[subplot_idx - 1], projection='3d')
ax.view_init(18, -70)
logger.debug(np.min(poses3d))
if np.min(poses3d) < -1000:
poses3d = before_pose
p3d = poses3d
logger.debug(poses3d)
viz.show3Dpose(p3d, ax, lcolor="#9b59b6", rcolor="#2ecc71")
pngName = FLAGS.output_dirname + '/' + FLAGS.output_filename + '_{0}.png'.format(
str(frame))
if FLAGS.write_output_img:
plt.savefig(pngName)
#png_lib.append(imageio.imread(pngName))
before_pose = poses3d
if FLAGS.write_gif:
logger.info("creating Gif movie_smoothing.gif, please Wait!")
imageio.mimsave(FLAGS.output_dirname + '/supplemental' +
'/movie_smoothing.gif',
png_lib,
fps=FLAGS.gif_fps)
logger.info("Done!".format(pngName))
def main(_):
smoothed = read_openpose_json()
'''plt.figure(2)
smooth_curves_plot = show_anim_curves(smoothed, plt)
#return
pngName = 'gif_output/smooth_plot.png'
smooth_curves_plot.savefig(pngName)
logger.info('writing gif_output/smooth_plot.png')'''
if FLAGS.interpolation:
logger.info("start interpolation")
framerange = len(smoothed.keys())
joint_rows = 36
array = np.concatenate(list(smoothed.values()))
array_reshaped = np.reshape(array, (framerange, joint_rows))
multiplier = FLAGS.multiplier
multiplier_inv = 1 / multiplier
out_array = np.array([])
for row in range(joint_rows):
x = []
for frame in range(framerange):
x.append(array_reshaped[frame, row])
frame = range(framerange)
frame_resampled = np.arange(0, framerange, multiplier)
spl = UnivariateSpline(frame, x, k=3)
#relative smooth factor based on jnt anim curve
min_x, max_x = min(x), max(x)
smooth_fac = max_x - min_x
smooth_resamp = 125
smooth_fac = smooth_fac * smooth_resamp
spl.set_smoothing_factor(float(smooth_fac))
xnew = spl(frame_resampled)
out_array = np.append(out_array, xnew)
logger.info(
"done interpolating. reshaping {0} frames, please wait!!".format(
framerange))
a = np.array([])
for frame in range(int(framerange * multiplier_inv)):
jnt_array = []
for jnt in range(joint_rows):
jnt_array.append(
out_array[jnt * int(framerange * multiplier_inv) + frame])
a = np.append(a, jnt_array)
a = np.reshape(a, (int(framerange * multiplier_inv), joint_rows))
out_array = a
interpolate_smoothed = {}
for frame in range(int(framerange * multiplier_inv)):
interpolate_smoothed[frame] = list(out_array[frame])
'''plt.figure(3)
smoothed = interpolate_smoothed
interpolate_curves_plot = show_anim_curves(smoothed, plt)
pngName = 'gif_output/interpolate_{0}.png'.format(smooth_resamp)
interpolate_curves_plot.savefig(pngName)
logger.info('writing gif_output/interpolate_plot.png')'''
enc_in = np.zeros((1, 64))
enc_in[0] = [0 for i in range(64)]
actions = data_utils.define_actions(FLAGS.action)
SUBJECT_IDS = [1, 5, 6, 7, 8, 9, 11]
rcams = cameras.load_cameras(FLAGS.cameras_path, SUBJECT_IDS)
train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.read_2d_predictions(
actions, FLAGS.data_dir)
train_set_3d, test_set_3d, data_mean_3d, data_std_3d, dim_to_ignore_3d, dim_to_use_3d, train_root_positions, test_root_positions = data_utils.read_3d_data(
actions, FLAGS.data_dir, FLAGS.camera_frame, rcams, FLAGS.predict_14)
device_count = {"GPU": 1}
png_lib = []
with tf.Session(config=tf.ConfigProto(device_count=device_count,
allow_soft_placement=True)) as sess:
#plt.figure(3)
batch_size = 128
model = create_model(sess, actions, batch_size)
iter_range = len(smoothed.keys())
rows = 0
filename = "keyjoints_proband_222_01.xlsx"
workbook = xlsxwriter.Workbook(filename)
worksheet = workbook.add_worksheet()
for n, (frame, xy) in enumerate(smoothed.items()):
logger.info("calc frame {0}/{1}".format(frame, iter_range))
# map list into np array
joints_array = np.zeros((1, 36))
joints_array[0] = [0 for i in range(36)]
for o in range(len(joints_array[0])):
#feed array with xy array
joints_array[0][o] = xy[o]
_data = joints_array[0]
# mapping all body parts or 3d-pose-baseline format
for i in range(len(order)):
for j in range(2):
# create encoder input
enc_in[0][order[i] * 2 + j] = _data[i * 2 + j]
for j in range(2):
# Hip
enc_in[0][0 * 2 + j] = (enc_in[0][1 * 2 + j] +
enc_in[0][6 * 2 + j]) / 2
# Neck/Nose
enc_in[0][14 * 2 + j] = (enc_in[0][15 * 2 + j] +
enc_in[0][12 * 2 + j]) / 2
# Thorax
enc_in[0][13 * 2 +
j] = 2 * enc_in[0][12 * 2 + j] - enc_in[0][14 * 2 +
j]
# set spine
spine_x = enc_in[0][24]
spine_y = enc_in[0][25]
enc_in = enc_in[:, dim_to_use_2d]
mu = data_mean_2d[dim_to_use_2d]
stddev = data_std_2d[dim_to_use_2d]
enc_in = np.divide((enc_in - mu), stddev)
dp = 1.0
dec_out = np.zeros((1, 48))
dec_out[0] = [0 for i in range(48)]
_, _, poses3d = model.step(sess,
enc_in,
dec_out,
dp,
isTraining=False)
all_poses_3d = []
enc_in = data_utils.unNormalizeData(enc_in, data_mean_2d,
data_std_2d, dim_to_ignore_2d)
poses3d = data_utils.unNormalizeData(poses3d, data_mean_3d,
data_std_3d, dim_to_ignore_3d)
gs1 = gridspec.GridSpec(1, 1)
gs1.update(wspace=-0.00,
hspace=0.05) # set the spacing between axes.
plt.axis('off')
all_poses_3d.append(poses3d)
enc_in, poses3d = map(np.vstack, [enc_in, all_poses_3d])
subplot_idx, exidx = 1, 1
_max = 0
_min = 10000
for i in range(poses3d.shape[0]):
for j in range(32):
tmp = poses3d[i][j * 3 + 2]
poses3d[i][j * 3 + 2] = poses3d[i][j * 3 + 1]
poses3d[i][j * 3 + 1] = tmp
if poses3d[i][j * 3 + 2] > _max:
_max = poses3d[i][j * 3 + 2]
if poses3d[i][j * 3 + 2] < _min:
_min = poses3d[i][j * 3 + 2]
for i in range(poses3d.shape[0]):
for j in range(32):
poses3d[i][j * 3 + 2] = _max - poses3d[i][j * 3 + 2] + _min
poses3d[i][j * 3] += (spine_x - 630)
poses3d[i][j * 3 + 2] += (500 - spine_y)
# Plot 3d predictions
#fig = plt.figure( figsize=(12.8, 7.2) )
ax = plt.subplot(gs1[subplot_idx - 1], projection='3d')
#ax.view_init(18, -70)
logger.debug(np.min(poses3d))
if np.min(poses3d) < -1000:
poses3d = before_pose
p3d = poses3d
logger.debug(poses3d)
#viz.Ax3DPose(ax, lcolor="#9b59b6", rcolor="#2ecc71")
viz.show3Dpose(p3d, ax, lcolor="#9b59b6", rcolor="#2ecc71")
#ax.scatter(p3d, c='b', marker='o')
#result = viz.show3Dpose(p3d,ax)
#print(p3d)
ax.set_xlabel('X Label')
ax.set_ylabel('Y Label')
ax.set_zlabel('Z Label')
ax.set_aspect('equal')
#print(p3d)
#plt.show()
col = 0
for i in p3d[0]:
worksheet.write(rows, col, i)
col += 1
#.append(i)
rows += 1
pngName = '{}_keypoints.png'.format(str(frame))
plt.savefig(pngName)
if FLAGS.write_gif:
png_lib.append(imageio.imread(pngName))
before_pose = poses3d
workbook.close()
if FLAGS.write_gif:
if FLAGS.interpolation:
#take every frame on gif_fps * multiplier_inv
png_lib = np.array([
png_lib[png_image]
for png_image in range(0, len(png_lib), int(multiplier_inv))
])
logger.info("creating Gif gif_output/animation.gif, please Wait!")
imageio.mimsave('gif_output/animation.gif', png_lib, fps=FLAGS.gif_fps)
logger.info("Done!".format(pngName))
def main(_):
actions_all = data_utils.define_actions( "All" )
# Load camera parameters
SUBJECT_IDS = [1,5,6,7,8,9,11]
rcams = cameras.load_cameras(FLAGS.cameras_path, SUBJECT_IDS)
# Load 3d data and load (or create) 2d projections
train_set_3d, test_set_3d, data_mean_3d, data_std_3d, dim_to_ignore_3d, dim_to_use_3d, train_root_positions, test_root_positions = data_utils.read_3d_data(
actions_all, FLAGS.data_dir, FLAGS.camera_frame, rcams, FLAGS.predict_14 )
print("Finished Read 3D Data")
# _, _, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.read_2d_predictions(actions_all, FLAGS.data_dir)
_, _, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.create_2d_data( actions_all, FLAGS.data_dir, rcams )
print("Finished Read 2D Data")
SH_TO_GT_PERM = np.array([SH_NAMES.index( h ) for h in H36M_NAMES if h != '' and h in SH_NAMES])
assert np.all( SH_TO_GT_PERM == np.array([6,2,1,0,3,4,5,7,8,9,13,14,15,12,11,10]) )
test_set = {}
manipulation_dir = os.path.dirname(FLAGS.data_dir)
manipulation_dir = os.path.dirname(manipulation_dir)
manipulation_dir += '/manipulation_video/'
manipulation_folders = glob.glob(manipulation_dir + '*')
subj = 1
action = 'manipulation-video'
for folder in manipulation_folders:
seqname = os.path.basename( folder )
with h5py.File(folder + '/' + seqname +'.h5', 'r' ) as h5f:
poses = h5f['poses'][:]
# Permute the loaded data to make it compatible with H36M
poses = poses[:,SH_TO_GT_PERM,:]
# Reshape into n x (32*2) matrix
poses = np.reshape(poses,[poses.shape[0], -1])
poses_final = np.zeros([poses.shape[0], len(H36M_NAMES)*2])
dim_to_use_x = np.where(np.array([x != '' and x != 'Neck/Nose' for x in H36M_NAMES]))[0] * 2
dim_to_use_y = dim_to_use_x+1
dim_to_use = np.zeros(len(SH_NAMES)*2,dtype=np.int32)
dim_to_use[0::2] = dim_to_use_x
dim_to_use[1::2] = dim_to_use_y
poses_final[:,dim_to_use] = poses
# If Uniframe Model Only keep the right 2d pose
poses_final = poses_final[1::2,:]
print(seqname, poses_final.shape)
test_set[ (subj, action, seqname) ] = poses_final
print("Finished Read Manipulations Videos")
test_set_2d = data_utils.normalize_data( test_set, data_mean_2d, data_std_2d, dim_to_use_2d )
dim_to_use_12_manipulation_joints = np.array([3,4,5, 6,7,8, 9,10,11, 18,19,20, 21,22,23, 24,25,26, 51,52,53, 54,55,56, 57,58,59, 75,76,77, 78,79,80, 81,82,83])
print("Finished Normalize Manipualtion Videos")
device_count = {"GPU": 0} if FLAGS.use_cpu else {"GPU": 1}
with tf.Session(config=tf.ConfigProto( device_count = device_count )) as sess:
# === Create the model ===
print("Creating %d layers of %d units." % (FLAGS.num_layers, FLAGS.linear_size))
batch_size = FLAGS.batch_size #Intial code is 64*2
model = predict_3dpose.create_model(sess, actions_all, batch_size)
print("Model loaded")
j = 0
for key2d in test_set_2d.keys():
(subj, b, fname) = key2d
# if fname != specific_seqname + '.h5':
# continue
print( "Subject: {}, action: {}, fname: {}".format(subj, b, fname) )
enc_in = test_set_2d[ key2d ]
n2d, _ = enc_in.shape
# Split into about-same-size batches
enc_in = np.array_split( enc_in, n2d // 1 )
all_poses_3d = []
for bidx in range( len(enc_in) ):
# Dropout probability 0 (keep probability 1) for sampling
dp = 1.0
anything = np.zeros((enc_in[bidx].shape[0], 48))
_, _, poses3d = model.step(sess, enc_in[bidx], anything, dp, isTraining=False)
# Denormalize
enc_in[bidx] = data_utils.unNormalizeData( enc_in[bidx], data_mean_2d, data_std_2d, dim_to_ignore_2d )
poses3d = data_utils.unNormalizeData( poses3d, data_mean_3d, data_std_3d, dim_to_ignore_3d )
all_poses_3d.append( poses3d )
# Put all the poses together
enc_in, poses3d = map( np.vstack, [enc_in, all_poses_3d] )
enc_in, poses3d = map( np.vstack, [enc_in, poses3d] )
poses3d_12_manipulation = poses3d[:, dim_to_use_12_manipulation_joints]
annotated_images = glob.glob(manipulation_dir + fname + '/info/*.xml')
annotated_images = sorted(annotated_images)
# 1080p = 1,920 x 1,080
fig = plt.figure(j, figsize=(10, 10))
gs1 = gridspec.GridSpec(3, 3)
gs1.update(wspace=-0, hspace=0.1) # set the spacing between axes.
plt.axis('off')
subplot_idx = 1
nsamples = 3
for i in np.arange( nsamples ):
# Plot 2d Detection
ax1 = plt.subplot(gs1[subplot_idx-1])
img = mpimg.imread(manipulation_dir + fname + '/skeleton_cropped/' + os.path.basename(annotated_images[i]).split('_')[0] + '.jpg')
ax1.imshow(img)
# Plot 2d pose
ax2 = plt.subplot(gs1[subplot_idx])
p2d = enc_in[i,:]
viz.show2Dpose( p2d, ax2 )
ax2.invert_yaxis()
# Plot 3d predictions
# Compute first the procrustion and print error
gt = getJ3dPosFromXML(annotated_images[i])
A = poses3d_12_manipulation[i, :].reshape(gt.shape)
_, Z, T, b, c = procrustes.compute_similarity_transform(gt, A, compute_optimal_scale=True)
sqerr = np.sqrt(np.sum((gt - (b*A.dot(T)) - c)**2, axis = 1))
print("{0} - {1} - Mean Error (mm) : {2}".format(fname, os.path.basename(annotated_images[i]), np.mean(sqerr)))
ax3 = plt.subplot(gs1[subplot_idx+1], projection='3d')
temp = poses3d[i,:].reshape((32, 3))
temp = c + temp.dot(T) #Do not scale
p3d = temp.reshape((1, 96))
viz.show3Dpose( p3d, ax3, lcolor="#9b59b6", rcolor="#2ecc71" )
ax3.invert_zaxis()
ax3.invert_yaxis()
subplot_idx = subplot_idx + 3
plt.show()
j += 1
def main(_):
enc_in = np.zeros((1, 64))
enc_in[0] = [0 for i in range(64)]
actions = data_utils.define_actions(FLAGS.action)
SUBJECT_IDS = [1, 5, 6, 7, 8, 9, 11]
rcams = cameras.load_cameras(FLAGS.cameras_path, SUBJECT_IDS)
train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.read_2d_predictions(
actions, FLAGS.data_dir)
train_set_3d, test_set_3d, data_mean_3d, data_std_3d, dim_to_ignore_3d, dim_to_use_3d, train_root_positions, test_root_positions = data_utils.read_3d_data(
actions, FLAGS.data_dir, FLAGS.camera_frame, rcams, FLAGS.predict_14)
device_count = {"GPU": 1}
png_lib = []
sess = tf.Session(config=tf.ConfigProto(
device_count=device_count,
allow_soft_placement=True))
# plt.figure(3)
batch_size = 128
model = create_model(sess, actions, batch_size)
''''''
openpose_dirs = sorted([ name for name in os.listdir(openpose_output_dir) if os.path.isdir(os.path.join(openpose_output_dir, name)) ])
openpose_dirs = [openpose_output_dir + file for file in openpose_dirs]
error_dirs = []
for dir in openpose_dirs:
# try:
file_name = os.path.basename(dir)
file_to_save = '../data/3d-pose-baseline/' + file_name + '.npz'
if not os.path.isfile(file_to_save): # If haven't already processed
smoothed = read_openpose_json(dir)
plt.figure(2)
# smooth_curves_plot = show_anim_curves(smoothed, plt)
# return
# pngName = 'gif_output/smooth_plot.png'
# smooth_curves_plot.savefig(pngName)
# logger.info('writing gif_output/smooth_plot.png')
if FLAGS.interpolation:
logger.info("start interpolation")
framerange = len(smoothed.keys())
joint_rows = 36
array = np.concatenate(list(smoothed.values()))
array_reshaped = np.reshape(array, (framerange, joint_rows))
multiplier = FLAGS.multiplier
multiplier_inv = 1 / multiplier
out_array = np.array([])
for row in range(joint_rows):
x = []
for frame in range(framerange):
x.append(array_reshaped[frame, row])
frame = range(framerange)
frame_resampled = np.arange(0, framerange, multiplier)
spl = UnivariateSpline(frame, x, k=3)
# relative smooth factor based on jnt anim curve
min_x, max_x = min(x), max(x)
smooth_fac = max_x - min_x
smooth_resamp = 125
smooth_fac = smooth_fac * smooth_resamp
spl.set_smoothing_factor(float(smooth_fac))
xnew = spl(frame_resampled)
out_array = np.append(out_array, xnew)
logger.info("done interpolating. reshaping {0} frames, please wait!!".format(framerange))
a = np.array([])
for frame in range(int(framerange * multiplier_inv)):
jnt_array = []
for jnt in range(joint_rows):
jnt_array.append(out_array[jnt * int(framerange * multiplier_inv) + frame])
a = np.append(a, jnt_array)
a = np.reshape(a, (int(framerange * multiplier_inv), joint_rows))
out_array = a
interpolate_smoothed = {}
for frame in range(int(framerange * multiplier_inv)):
interpolate_smoothed[frame] = list(out_array[frame])
# plt.figure(3)
smoothed = interpolate_smoothed
# interpolate_curves_plot = show_anim_curves(smoothed, plt)
# pngName = 'gif_output/interpolate_{0}.png'.format(smooth_resamp)
# interpolate_curves_plot.savefig(pngName)
# logger.info('writing gif_output/interpolate_plot.png')
iter_range = len(smoothed.keys())
positions_3d = []
for n, (frame, xy) in enumerate(smoothed.items()):
logger.info("calc frame {0}/{1}".format(frame, iter_range))
# map list into np array
joints_array = np.zeros((1, 36))
joints_array[0] = [0 for i in range(36)]
for o in range(len(joints_array[0])):
#feed array with xy array
joints_array[0][o] = xy[o]
_data = joints_array[0]
# mapping all body parts or 3d-pose-baseline format
for i in range(len(order)):
for j in range(2):
# create encoder input
enc_in[0][order[i] * 2 + j] = _data[i * 2 + j]
for j in range(2):
# Hip
enc_in[0][0 * 2 + j] = (enc_in[0][1 * 2 + j] + enc_in[0][6 * 2 + j]) / 2
# Neck/Nose
enc_in[0][14 * 2 + j] = (enc_in[0][15 * 2 + j] + enc_in[0][12 * 2 + j]) / 2
# Thorax
enc_in[0][13 * 2 + j] = 2 * enc_in[0][12 * 2 + j] - enc_in[0][14 * 2 + j]
# set spine
spine_x = enc_in[0][24]
spine_y = enc_in[0][25]
enc_in = enc_in[:, dim_to_use_2d]
mu = data_mean_2d[dim_to_use_2d]
stddev = data_std_2d[dim_to_use_2d]
enc_in = np.divide((enc_in - mu), stddev)
dp = 1.0
dec_out = np.zeros((1, 48))
dec_out[0] = [0 for i in range(48)]
_, _, poses3d = model.step(sess, enc_in, dec_out, dp, isTraining=False)
all_poses_3d = []
enc_in = data_utils.unNormalizeData(enc_in, data_mean_2d, data_std_2d, dim_to_ignore_2d)
poses3d = data_utils.unNormalizeData(poses3d, data_mean_3d, data_std_3d, dim_to_ignore_3d)
gs1 = gridspec.GridSpec(1, 1)
gs1.update(wspace=-0.00, hspace=0.05) # set the spacing between axes.
plt.axis('off')
all_poses_3d.append( poses3d )
enc_in, poses3d = map( np.vstack, [enc_in, all_poses_3d] )
subplot_idx, exidx = 1, 1
_max = 0
_min = 10000
for i in range(poses3d.shape[0]):
for j in range(32):
tmp = poses3d[i][j * 3 + 2]
poses3d[i][j * 3 + 2] = poses3d[i][j * 3 + 1]
poses3d[i][j * 3 + 1] = tmp
if poses3d[i][j * 3 + 2] > _max:
_max = poses3d[i][j * 3 + 2]
if poses3d[i][j * 3 + 2] < _min:
_min = poses3d[i][j * 3 + 2]
for i in range(poses3d.shape[0]):
for j in range(32):
poses3d[i][j * 3 + 2] = _max - poses3d[i][j * 3 + 2] + _min
poses3d[i][j * 3] += (spine_x - 630)
poses3d[i][j * 3 + 2] += (500 - spine_y)
# Plot 3d predictions
ax = plt.subplot(gs1[subplot_idx - 1], projection='3d')
ax.view_init(18, -70)
logger.debug(np.min(poses3d))
if np.min(poses3d) < -1000 and frame != 0:
poses3d = before_pose
p3d = poses3d
logger.debug(poses3d)
viz.show3Dpose(p3d, ax, lcolor="#9b59b6", rcolor="#2ecc71")
positions_3d.append(p3d)
pngName = 'png/pose_frame_{0}.png'.format(str(frame).zfill(12))
plt.savefig(pngName)
if FLAGS.write_gif:
png_lib.append(imageio.imread(pngName))
before_pose = poses3d
np.savez_compressed(file_to_save, positions_3d=positions_3d)
if FLAGS.write_gif:
if FLAGS.interpolation:
#take every frame on gif_fps * multiplier_inv
png_lib = np.array([png_lib[png_image] for png_image in range(0,len(png_lib), int(multiplier_inv)) ])
logger.info("creating Gif png/animation_" + file_name + ".gif, please Wait!")
imageio.mimsave("gif_output/animation_" + file_name + ".gif", png_lib, fps=FLAGS.gif_fps)
logger.info("Done!".format(pngName))
# except:
# error_dirs.append(dir)
with open('error_op_3d_dirs.txt', 'w') as f:
for item in error_dirs:
f.write("%s\n" % item)
