def extract_annotated_scenelet(
scene,
prefix_obj='obb',
frame_ids=None,
frame_multiplier=1.,
time_multiplier=1.,
f_ob_is_joint=lambda ob: ob.name.startswith(
'Output') and ob.name.endswith('Sphere'),
f_joint_name_from_ob=lambda ob: ob.name.split('.')[1]):
"""
Args:
scene (bpy.types.Scene):
The current scene (e.g. bpy.context.scene).
prefix_obj (str):
Start of object names that we want to include in the scenelet
as oriented bounding boxes.
frame_ids (List[int]):
A subset of frame IDs to export.
frame_multiplier (float):
Scaling for frame IDs. The result will be rounded and truncated.
output.frame_id := int(round(frame_id * frame_multiplier))
time_multipler (float):
Scaling for times associated with frame_ids.
output.time := int(round(frame_id * frame_multiplier))
* time_multiplier.
f_ob_is_joint (Callable[[bpy.types.Object], bool]]):
Decides if a Blender object is a joint.
f_joint_name_from_ob (Callable[[bpy.types.Object], str]):
Gets the joint name from the Blender object name.
"""
# joints = {
# ob.name.split('.')[1]: ob
# for ob in bpy.data.objects
# if ob.name.startswith('Output') and ob.name.endswith('Sphere')}
joints = {
f_joint_name_from_ob(ob): ob
for ob in bpy.data.objects if f_ob_is_joint(ob)
}
print("joints: %s" % joints)
skeleton = Skeleton()
if len(joints):
assert len(joints) == 16, "No: %s" % len(joints)
if not frame_ids:
frame_ids = range(scene.frame_start, scene.frame_end + 1)
for frame_id in frame_ids:
o_frame_id = int(round(frame_id * frame_multiplier))
if skeleton.has_pose(o_frame_id):
print("skipping %s" % frame_id)
continue
print("frame_id: %s" % frame_id)
scene.frame_set(frame_id)
bpy.context.scene.update()
# bpy.ops.anim.change_frame(frame_id)
pose = np.zeros(shape=(3, len(joints)))
for joint, ob in joints.items():
pos = ob.matrix_world.col[3]
print("pos[%s]: %s" % (ob.name, pos))
joint_id = Joint.from_string(joint)
print("joint %s is %s" % (joint, Joint(joint_id)))
pose[:, joint_id] = from_blender(pos)
print("o_frame: %s from %s" % (o_frame_id, frame_id))
assert not skeleton.has_pose(o_frame_id), \
"Already has %s" % frame_id
skeleton.set_pose(frame_id=o_frame_id,
pose=pose,
time=o_frame_id * time_multiplier)
objs_bl = {}
for obj in bpy.data.objects:
if obj.name.startswith(prefix_obj) and not obj.hide:
obj_id = int(obj.name.split('_')[1])
try:
objs_bl[obj_id].append(obj)
except KeyError:
objs_bl[obj_id] = [obj]
print("objs: %s" % objs_bl)
scenelet = Scenelet(skeleton=skeleton)
print("scenelet: %s" % scenelet)
for obj_id, parts_bl in objs_bl.items():
name_category = None
scene_obj = None
for part_id, part_bl in enumerate(parts_bl):
transl, rot, scale = part_bl.matrix_world.decompose()
rot = rot.to_matrix()
if any(comp < 0. for comp in scale):
scale *= -1.
rot *= -1.
assert not any(comp < 0. for comp in scale), "No: %s" % scale
matrix_world = part_bl.matrix_world.copy()
# need to save full scale, not only half axes
for c in range(3):
for r in range(3):
matrix_world[r][c] *= 2.
name_parts = part_bl.name.split('_')
if name_category is None:
name_category = name_parts[2]
scene_obj = SceneObj(label=name_category)
else:
assert name_category == name_parts[2], \
"No: %s %s" % (name_category, name_parts[2])
name_part = name_parts[3]
print("part: %s" % name_part)
part = SceneObjPart(name_part)
part.obb = Obb(centroid=np.array(
from_blender([transl[0], transl[1], transl[2]])),
axes=np.array([[rot[0][0], rot[0][1], rot[0][2]],
[-rot[2][0], -rot[2][1], -rot[2][2]],
[rot[1][0], rot[1][1], rot[1][2]]]),
scales=np.array(
[scale[0] * 2., scale[1] * 2., scale[2] * 2.]))
# if 'table' in name_category:
# print(part.obb.axes)
# raise RuntimeError("stop")
print("obb: %s" % part.obb.to_json(0))
scene_obj.add_part(part_id, part)
scenelet.add_object(obj_id, scene_obj, clone=False)
return scenelet
def match(query_full,
d_query,
query_2d_full,
scene,
intr,
gap,
tr_ground,
scale,
thresh_log_conf=7.5,
w_3d=0.01,
fps=3,
step_samples=100):
with_y = False # optimize for y as well
np.set_printoptions(suppress=True, linewidth=220)
pjoin = os.path.join
len_gap = gap[1] - gap[0] + 1
query, q_v = get_partial_scenelet(query_full,
start=gap[0],
end=gap[1] + 1,
fps=1)
q_v_sum = np.sum(q_v)
q_v_sum_inv = np.float32(1. / q_v_sum)
# lg.debug("q_v_sum: %s/%s" % (q_v_sum, q_v.size))
# scene_min_y = scene.skeleton.get_min_y(tr_ground)
# lg.debug("scene_min_y: %s" % repr(scene_min_y))
mid_frames = range(len_gap * fps,
scene.skeleton.poses.shape[0] - len_gap * fps,
step_samples)
if not len(mid_frames):
return []
scenelets, sc_v = (np.array(e) for e in zip(*[
get_partial_scenelet(
scene, mid_frame_id=mid_frame_id, n_frames=len_gap, fps=fps)
for mid_frame_id in mid_frames
]))
# for i, (scenelet, sc_v_) in enumerate(zip(scenelets, sc_v)):
# mn = np.min(scenelet[sc_v_.astype('b1'), 1, :])
# scenelets[i, :, 1, :] -= mn
# mn = np.min(scenelets[i, sc_v_.astype('b1'), 1, :])
# scenelets = np.array(scenelets, dtype=np.float32)
# sc_v = np.array(sc_v, dtype=np.int32)
# print("sc_v: %s" % sc_v)
# print("q_v: %s" % q_v)
lg.debug("have %d/%d 3D poses in scenelet, and %d/%d in query" %
(np.sum(sc_v), sc_v.shape[0], np.sum(q_v), q_v.shape[0]))
query_2d = np.zeros((len_gap, 2, 16), dtype=np.float32)
conf_2d = np.zeros((len_gap, 1, 16), dtype=np.float32)
for lin_id, frame_id in enumerate(range(gap[0], gap[1] + 1)):
if query_2d_full.has_pose(frame_id):
query_2d[lin_id, :, :] = query_2d_full.get_pose(frame_id)[:2, :]
# else:
# lg.warning("Query2d_full does not have pose at %d?" % frame_id)
# im = im_.copy()
if query_2d_full.has_confidence(frame_id):
# print("showing %s" % frame_id)
for joint, conf in query_2d_full._confidence[frame_id].items():
log_conf = abs(np.log(conf)) if conf >= 0. else 0.
# print("conf: %g, log_conf: %g" % (conf, log_conf))
# if log_conf <= thresh_log_conf:
# p2d = scale * query_2d_full.get_joint_3d(joint,
# frame_id=frame_id)
# p2d = (int(round(p2d[0])), int(round(p2d[1])))
# cv2.circle(im, center=p2d,
# radius=int(round(3)),
# color=(1., 1., 1., 0.5), thickness=1)
conf_2d[lin_id, 0, joint] = max(
0., (thresh_log_conf - log_conf) / thresh_log_conf)
# cv2.imshow('im', im)
# cv2.waitKey(100)
# while cv2.waitKey() != 27: pass
conf_2d /= np.max(conf_2d)
# scale from Denis' scale to current image size
query_2d *= scale
# move to normalized camera coordinates
query_2d -= intr[:2, 2:3]
query_2d[:, 0, :] /= intr[0, 0]
query_2d[:, 1, :] /= intr[1, 1]
#
# initialize translation
#
# centroid of query poses
c3d = np.mean(query[q_v.astype('b1'), :, :], axis=(0, 2))
# estimate scenelet centroids
sclt_means = np.array([
np.mean(scenelets[i, sc_v[i, ...].astype('b1'), ...], axis=(0, 2))
for i in range(scenelets.shape[0])
],
dtype=np.float32)
# don't change height
sclt_means[:, 1] = 0
scenelets -= sclt_means[:, None, :, None]
lg.debug("means: %s" % repr(sclt_means.shape))
if with_y:
np_translation = np.array([c3d for i in range(scenelets.shape[0])],
dtype=np.float32)
else:
np_translation = np.array(
[c3d[[0, 2]] for i in range(scenelets.shape[0])], dtype=np.float32)
np_rotation = np.array(
[np.pi * (i % 2) for i in range(scenelets.shape[0])],
dtype=np.float32)[:, None]
n_cands = np_translation.shape[0]
graph = tf.Graph()
with graph.as_default(), tf.device('/gpu:0'):
# 3D translation
translation_ = tf.Variable(initial_value=np_translation,
name='translation',
dtype=tf.float32)
t_y = tf.fill(dims=(n_cands, ),
value=(tr_ground[1, 3]).astype(np.float32))
# t_y = tf.fill(dims=(n_cands,), value=np.float32(0.))
lg.debug("t_y: %s" % t_y)
if with_y:
translation = translation_
else:
translation = tf.concat(
(translation_[:, 0:1], t_y[:, None], translation_[:, 1:2]),
axis=1)
lg.debug("translation: %s" % translation)
# 3D rotation (Euler XYZ)
rotation = tf.Variable(np_rotation, name='rotation', dtype=tf.float32)
# lg.debug("rotation: %s" % rotation)
w = tf.Variable(conf_2d, trainable=False, name='w', dtype=tf.float32)
pos_3d_in = tf.Variable(query,
trainable=False,
name='pos_3d_in',
dtype=tf.float32)
# pos_3d_in = tf.constant(query, name='pos_3d_in', dtype=tf.float32)
pos_2d_in = tf.Variable(query_2d,
trainable=False,
name='pos_2d_in',
dtype=tf.float32)
# pos_2d_in = tf.constant(query_2d, name='pos_2d_in',
# dtype=tf.float32)
pos_3d_sclt = tf.Variable(scenelets,
trainable=False,
name='pos_3d_sclt',
dtype=tf.float32)
# print("pos_3d_sclt: %s" % pos_3d_sclt)
# rotation around y
my_zeros = tf.zeros((n_cands, 1), dtype=tf.float32, name='my_zeros')
# tf.add_to_collection('to_init', my_zeros)
my_ones = tf.ones((n_cands, 1))
# tf.add_to_collection('to_init', my_ones)
c = tf.cos(rotation, 'cos')
# tf.add_to_collection('to_init', c)
s = tf.sin(rotation, 'sin')
# t0 = tf.concat([c, my_zeros, -s], axis=1)
# t1 = tf.concat([my_zeros, my_ones, my_zeros], axis=1)
# t2 = tf.concat([s, my_zeros, c], axis=1)
# transform = tf.stack([t0, t1, t2], axis=2, name="transform")
# print("t: %s" % transform)
transform = tf.concat(
[c, my_zeros, -s, my_zeros, my_ones, my_zeros, s, my_zeros, c],
axis=1)
transform = tf.reshape(transform, ((-1, 3, 3)), name='transform')
print("t2: %s" % transform)
# lg.debug("transform: %s" % transform)
# transform to 3d
# pos_3d = tf.matmul(transform, pos_3d_sclt) \
# + tf.tile(tf.expand_dims(translation, 2),
# [1, 1, int(pos_3d_in.shape[2])])
# pos_3d = tf.einsum("bjk,bcjd->bcjd", transform, pos_3d_sclt)
shp = pos_3d_sclt.get_shape().as_list()
transform_tiled = tf.tile(transform[:, None, :, :, None],
(1, shp[1], 1, 1, shp[3]))
# print("transform_tiled: %s" % transform_tiled)
pos_3d = tf.einsum("abijd,abjd->abid", transform_tiled, pos_3d_sclt)
# print("pos_3d: %s" % pos_3d)
pos_3d += translation[:, None, :, None]
#pos_3d = pos_3d_sclt
# print("pos_3d: %s" % pos_3d)
# perspective divide
# pos_2d = tf.divide(
# tf.slice(pos_3d, [0, 0, 0], [n_cands, 2, -1]),
# tf.slice(pos_3d, [0, 2, 0], [n_cands, 1, -1]))
pos_2d = tf.divide(pos_3d[:, :, :2, :], pos_3d[:, :, 2:3, :])
# print("pos_2d: %s" % pos_2d)
diff = pos_2d - pos_2d_in
# mask loss by 2d key-point visibility
# print("w: %s" % w)
# w_sum = tf.reduce_sum()
masked = tf.multiply(diff, w)
# print(masked)
# loss_reproj = tf.nn.l2_loss(masked)
# loss_reproj = tf.reduce_sum(tf.square(masked[:, :, 0, :])
# + tf.square(masked[:, :, 1, :]),
# axis=[1, 2])
masked_sqr = tf.square(masked[:, :, 0, :]) \
+ tf.square(masked[:, :, 1, :])
loss_reproj = tf.reduce_sum(masked_sqr, axis=[1, 2])
# lg.debug("loss_reproj: %s" % loss_reproj)
# distance from existing 3D skeletons
d_3d = q_v_sum_inv * tf.multiply(pos_3d - query[None, ...],
q_v[None, :, None, None],
name='diff_3d')
# print(d_3d)
loss_3d = w_3d * tf.reduce_sum(tf.square(d_3d[:, :, 0, :]) + tf.square(
d_3d[:, :, 1, :]) + tf.square(d_3d[:, :, 2, :]),
axis=[1, 2],
name='loss_3d_each')
# print(loss_3d)
loss = tf.reduce_sum(loss_reproj) + tf.reduce_sum(loss_3d)
# optimize
optimizer = ScipyOptimizerInterface(loss,
var_list=[translation_, rotation],
options={'gtol': 1e-12})
with Timer('solve', verbose=True) as t:
with tf.Session(graph=graph) as session:
session.run(tf.global_variables_initializer())
optimizer.minimize(session)
o_pos_3d, o_pos_2d, o_masked, o_t, o_r, o_w, o_d_3d, \
o_loss_reproj, o_loss_3d, o_transform, o_translation = \
session.run([
pos_3d, pos_2d, masked, translation, rotation, w,
d_3d, loss_reproj, loss_3d, transform, translation])
o_masked_sqr = session.run(masked_sqr)
# o_t, o_r = session.run([translation, rotation])
# print("pos_3d: %s" % o_pos_3d)
# print("pos_2d: %s" % o_pos_2d)
# print("o_loss_reproj: %s, o_loss_3d: %s" % (o_loss_reproj, o_loss_3d))
# print("t: %s" % o_t)
# print("r: %s" % o_r)
chosen = sorted((i for i in range(o_loss_reproj.shape[0])),
key=lambda i2: o_loss_reproj[i2] + o_loss_3d[i2])
lg.info("Best candidate is %d with error %g + %g" %
(chosen[0], o_loss_reproj[chosen[0]], o_loss_3d[chosen[0]]))
# print("masked: %s" % o_masked)
# opp = np.zeros_like(o_pos_3d)
# for i in range(o_pos_3d.shape[0]):
# for j in range(o_pos_3d.shape[1]):
# for k in range(16):
# opp[i, j, :2, k] = o_pos_3d[i, j, :2, k] / o_pos_3d[i, j, 2:3, k]
# # opp[i, j, 0, k] *= intr[0, 0]
# # opp[i, j, 1, k] *= intr[1, 1]
# # opp[i, j, :2, k] *= intr[1, 1]
# a = o_pos_2d[i, j, :, k]
# b = opp[i, j, :2, k]
# if not np.allclose(a, b):
# print("diff: %s, %s" % (a, b))
o_pos_2d[:, :, 0, :] *= intr[0, 0]
o_pos_2d[:, :, 1, :] *= intr[1, 1]
o_pos_2d += intr[:2, 2:3]
# for cand_id in range(o_pos_2d.shape[0]):
if False:
# return
# print("w: %s" % o_w)
# print("conf_2d: %s" % conf_2d)
# lg.debug("query_2d[0, 0, ...]: %s" % query_2d[0, 0, ...])
query_2d[:, 0, :] *= intr[0, 0]
query_2d[:, 1, :] *= intr[1, 1]
# lg.debug("query_2d[0, 0, ...]: %s" % query_2d[0, 0, ...])
query_2d += intr[:2, 2:3]
# lg.debug("query_2d[0, 0, ...]: %s" % query_2d[0, 0, ...])
ims = {}
for cand_id in chosen[:5]:
lg.debug("starting %s" % cand_id)
pos_ = o_pos_2d[cand_id, ...]
for lin_id in range(pos_.shape[0]):
frame_id = gap[0] + lin_id
try:
im = ims[frame_id].copy()
except KeyError:
p_im = pjoin(d_query, 'origjpg',
"color_%05d.jpg" % frame_id)
ims[frame_id] = cv2.imread(p_im)
im = ims[frame_id].copy()
# im = im_.copy()
for jid in range(pos_.shape[-1]):
xy2 = int(round(query_2d[lin_id, 0, jid])), \
int(round(query_2d[lin_id, 1, jid]))
# print("printing %s" % repr(xy))
cv2.circle(im,
center=xy2,
radius=5,
color=(10., 200., 10.),
thickness=-1)
if o_masked[cand_id, lin_id, 0, jid] > 0 \
or o_w[lin_id, 0, jid] > 0:
xy = int(round(pos_[lin_id, 0, jid])), \
int(round(pos_[lin_id, 1, jid]))
# print("printing %s" % repr(xy))
cv2.circle(im,
center=xy,
radius=3,
color=(200., 10., 10.),
thickness=-1)
cv2.putText(im,
"d2d: %g" %
o_masked_sqr[cand_id, lin_id, jid],
org=((xy2[0] - xy[0]) // 2 + xy[0],
(xy2[1] - xy[1]) // 2 + xy[1]),
fontFace=1,
fontScale=1,
color=(0., 0., 0.))
cv2.line(im, xy, xy2, color=(0., 0., 0.))
d3d = o_d_3d[cand_id, lin_id, :, jid]
d3d_norm = np.linalg.norm(d3d)
if d3d_norm > 0.:
cv2.putText(
im,
"%g" % d3d_norm,
org=((xy2[0] - xy[0]) // 2 + xy[0] + 10,
(xy2[1] - xy[1]) // 2 + xy[1]),
fontFace=1,
fontScale=1,
color=(0., 0., 255.))
cv2.putText(im,
text="%d::%02d" % (cand_id, lin_id),
org=(40, 80),
fontFace=1,
fontScale=2,
color=(255., 255., 255.))
# pos_2d_ = np.matmul(intr, pos_[lin_id, :2, :] / pos_[lin_id, 2:3, :])
# for p2d in pos_2d_
cv2.imshow('im', im)
cv2.waitKey()
break
while cv2.waitKey() != 27:
pass
out_scenelets = []
for cand_id in chosen[:1]:
lg.debug("score of %d is %g + %g = %g" %
(cand_id, o_loss_reproj[cand_id], o_loss_3d[cand_id],
o_loss_reproj[cand_id] + o_loss_3d[cand_id]))
scenelet = Scenelet()
rate = query_full.skeleton.get_rate()
prev_time = None
for lin_id, frame_id in enumerate(range(gap[0], gap[1] + 1)):
time_ = query_full.get_time(frame_id)
if lin_id and rate is None:
rate = time_ - prev_time
if time_ == frame_id:
time_ = prev_time + rate
scenelet.skeleton.set_pose(frame_id=frame_id,
pose=o_pos_3d[cand_id, lin_id, :, :],
time=time_)
prev_time = time_
tr = np.concatenate((np.concatenate(
(o_transform[cand_id, ...], o_translation[cand_id, None, :].T),
axis=1), [[0., 0., 0., 1.]]),
axis=0)
tr_m = np.concatenate(
(np.concatenate((np.identity(3), -sclt_means[cand_id, None, :].T),
axis=1), [[0., 0., 0., 1.]]),
axis=0)
tr = np.matmul(tr, tr_m)
for oid, ob in scene.objects.items():
if ob.label in ('wall', 'floor'):
continue
ob2 = copy.deepcopy(ob)
ob2.apply_transform(tr)
scenelet.add_object(obj_id=oid, scene_obj=ob2, clone=False)
scenelet.name_scene = scene.name_scene
out_scenelets.append((o_loss_reproj[cand_id], scenelet))
return out_scenelets
def main(argv):
parser = argparse.ArgumentParser(
"Filter initial path based on distance to full fit")
parser.add_argument('skel', help="Skeleton file to filter", type=str)
parser.add_argument('--threshold',
help='Distance threshold. Default: 0.4',
type=float,
default=0.4)
args = parser.parse_args(argv)
lower_body = [
Joint.LKNE, Joint.RKNE, Joint.LANK, Joint.RANK, Joint.LHIP, Joint.RHIP
]
print(args.skel)
p_root = os.path.dirname(args.skel)
p_fit = os.path.join(p_root, 'opt1')
assert os.path.isdir(p_fit), p_fit
query = Scenelet.load(args.skel)
out = Skeleton()
data = []
x = []
y = []
y2 = []
for d_ in sorted(os.listdir(p_fit)):
d = os.path.join(p_fit, d_)
pattern = os.path.join(d, 'skel_*.json')
for f in sorted(glob.iglob(pattern)):
print(f)
assert '00' in f, f
sclt = Scenelet.load(f)
frames = sclt.skeleton.get_frames()
mid_frame = frames[len(frames) // 2]
time = sclt.skeleton.get_time(mid_frame)
q_frame_id = query.skeleton.find_time(time)
q_time = query.skeleton.get_time(q_frame_id)
print(time, q_time, f)
q_pose = query.skeleton.get_pose(q_frame_id)
pose = sclt.skeleton.get_pose(mid_frame)
pose[[0, 2]] -= (pose[:, Joint.PELV:Joint.PELV + 1] -
q_pose[:, Joint.PELV:Joint.PELV + 1])[[0, 2]]
diff = np.mean(
np.linalg.norm(q_pose[:, lower_body] - pose[:, lower_body],
axis=0))
print(q_frame_id, time, diff)
y.append(diff)
x.append(q_frame_id)
data.append((q_frame_id, diff, time))
if query.skeleton.has_pose(q_frame_id - 1):
tmp_pose = copy.deepcopy(q_pose)
tmp_pose -= tmp_pose[:, Joint.PELV:Joint.PELV +
1] - query.skeleton.get_pose(
q_frame_id -
1)[:, Joint.PELV:Joint.PELV + 1]
y2.append(
np.mean(
np.linalg.norm(pose[:, lower_body] -
tmp_pose[:, lower_body],
axis=0)))
else:
y2.append(0.)
out.set_pose(frame_id=q_frame_id, time=q_time, pose=pose)
break
data = smooth(data)
plt.plot(x, y, 'x--', label='Distance to best Kinect fit\'s center frame')
plt.plot(x, y2, 'o--', label='Distance to prev pose')
plt.plot([d[0] for d in data], [d[1] for d in data],
'o--',
label='Smoothed')
plt.xlabel('Time (s)')
plt.ylabel('Sum local squared distance')
plt.legend()
plt.savefig(os.path.join(p_root, 'tmp.pdf'))
Scenelet(skeleton=out).save(os.path.join(p_root, 'skel_tmp.json'))
above = []
prev_frame_id = None
for frame_id, dist, time in data:
# assert prev_frame_id is None or frame_id != prev_frame_id, \
# 'No: {}'.format(frame_id)
if dist > args.threshold:
above.append(
Span2(start=frame_id, end=frame_id, value=dist, time=time))
prev_frame_id = frame_id
spans = [copy.deepcopy(above[0])]
it = iter(above)
next(it)
prev_frame_id = above[0].start
for span2 in it:
frame_id = span2.start
if prev_frame_id + 1 < frame_id:
# span = spans[-1]
# spans[-1] = span[0], prev_frame_id, span[2]
spans[-1].end = prev_frame_id
spans.append(
Span2(start=frame_id,
end=frame_id,
time=None,
value=span2.value))
else:
print(prev_frame_id, frame_id)
prev_frame_id = frame_id
spans[-1].end = prev_frame_id
print("Need replacement: {}".format(above))
print("Need replacement2: {}".format(spans))
def export_scenelet(um,
o_pos_3d,
o_polys_3d,
query_full_skeleton,
scenes,
joints_active,
transform_id=None):
"""Extract a scenelet (poses and objects) from the data from the
optimized problem.
Args:
um (stealth.pose.unk_manager.UnkManager):
Data manager.
o_pos_3d (np.ndarray):
Output 3D poses.
o_polys_3d (np.ndarray): (6K, 4, 3)
3D oriented bounding boxes stored stacked.
query_full_skeleton (stealth.logic.skeleton.Skeleton):
Initial path containing time information.
joints_active (list):
List of joint_ids that were optimized for.
Usage: pose16[:, joints_active] = o_pos_3d[pid, :, :]
transform_id (int):
Export only a specific group. Everything is exported, if None.
Returns:
A scenelet extracted from the data provided.
"""
# cache function
_guess_time_at = query_full_skeleton.guess_time_at
# all poses or the ones that belong to a group/scenelet
if transform_id is None:
pids_sorted = sorted([(pid, pid2scene)
for pid, pid2scene in um.pids_2_scenes.items()],
key=lambda e: e[1].frame_id)
else:
# pids_sorted = sorted([(pid, pid2scene)
# for pid, pid2scene in um.pids_2_scenes.items()
# if pid2scene.transform_id == transform_id],
# key=lambda e: e[1].frame_id)
pids_2_scenes = um.pids_2_scenes
pids_sorted = sorted([(pid, pids_2_scenes[pid])
for pid in um.get_pids_for(transform_id)],
key=lambda e: e[1].frame_id)
# create output scenelet
o = Scenelet()
charness = None
#
# Skeleton
#
# cache skeleton reference
skeleton = o.skeleton
# fill skeleton
for pid, pid2scene in pids_sorted:
if charness is None:
scene = scenes[pid2scene.id_scene]
charness = scene.charness
o.add_aux_info('name_scenelet', scene.name_scenelet)
o.charness = charness
# get frame_id
frame_id = int(pid2scene.frame_id)
# check if already exists
if skeleton.has_pose(frame_id):
# TODO: fix overlapping frame_ids
lg.warning("[export_scenelet] Overwriting output frame_id %d" %
frame_id)
# add with time guessed from input skeleton rate
pose = np.zeros((3, Joint.get_num_joints()))
pose[:, joints_active] = o_pos_3d[pid, :, :]
pose[:, Joint.PELV] = (pose[:, Joint.LHIP] + pose[:, Joint.RHIP]) / 2.
pose[:, Joint.NECK] = (pose[:, Joint.HEAD] + pose[:, Joint.THRX]) / 2.
# for j, jid in joints_remap.items():
# pose[:, j] = o_pos_3d[pid, :, jid]
assert not skeleton.has_pose(frame_id=frame_id), \
'Already has pose: {}'.format(frame_id)
skeleton.set_pose(frame_id=frame_id,
pose=pose,
time=_guess_time_at(frame_id))
#
# Objects
#
scene_obj = None
scene_obj_oid = 0 # unique identifier that groups parts to objects
for polys2scene in um.polys2scene.values():
# Check, if we are restricted to a certain group
if transform_id is not None \
and polys2scene.transform_id != transform_id:
continue
start = polys2scene.poly_id_start
end = start + polys2scene.n_polys
# 6 x 4 x 3
polys = o_polys_3d[start:end, ...]
assert polys.shape[0] == 6, "Assumed cuboids here"
if scene_obj is None or scene_obj_oid != polys2scene.object_id:
category = next(cat for cat in CATEGORIES
if CATEGORIES[cat] == polys2scene.cat_id)
scene_obj = SceneObj(label=category)
scene_obj_oid = polys2scene.object_id
o.add_object(obj_id=-1, scene_obj=scene_obj, clone=False)
part = scene_obj.add_part(part_id=-1,
label_or_part=polys2scene.part_label)
# TODO: average for numerical precision errors
centroid = np.mean(polys, axis=(0, 1))
ax0 = polys[0, 1, :] - polys[0, 0, :]
scale0 = np.linalg.norm(ax0)
ax0 /= scale0
ax1 = polys[0, 3, :] - polys[0, 0, :]
scale1 = np.linalg.norm(ax1)
ax1 /= scale1
ax2 = polys[1, 0, :] - polys[0, 0, :]
scale2 = np.linalg.norm(ax2)
ax2 /= scale2
part.obb = Obb(centroid=centroid,
axes=np.concatenate(
(ax0[:, None], ax1[:, None], ax2[:, None]), axis=1),
scales=[scale0, scale1, scale2])
# if scene_obj is not None:
# o.add_object(obj_id=-1, scene_obj=scene_obj, clone=False)
# else:
# lg.warning("No objects in scenelet?")
# scene_obj = SceneObj('couch')
# for poly_id in range(0, o_polys_3d.shape[0], 6):
# rects = o_polys_3d[poly_id : poly_id + 6, ...]
# # lg.debug("rects:\n%s" % rects)
# scene_obj.add_part(poly_id, 'seat')
#
# # fig = plt.figure()
# # ax = fig.add_subplot(111, projection='3d')
# # for rid, rect in enumerate(rects):
# # wrapped = np.concatenate((rect, rect[0:1, :]), axis=0)
# # ax.plot(wrapped[:, 0], wrapped[:, 2], wrapped[:, 1])
# # for ci in range(4):
# # c = rect[ci, :]
# # ax.text(c[0], c[2], c[1], s="%d, %d, %d"
# # % (poly_id, rid, ci))
# # if rid >= 1:
# # break
# #
# # plt.show()
# part = scene_obj.get_part(poly_id)
# centroid = np.mean(rects, axis=(0, 1))
# ax0 = rects[0, 1, :] - rects[0, 0, :]
# scale0 = np.linalg.norm(ax0)
# ax0 /= scale0
# ax1 = rects[0, 3, :] - rects[0, 0, :]
# scale1 = np.linalg.norm(ax1)
# ax1 /= scale1
# ax2 = rects[1, 0, :] - rects[0, 0, :]
# scale2 = np.linalg.norm(ax2)
# ax2 /= scale2
# part.obb = Obb(centroid=centroid,
# axes=np.concatenate((
# ax0[:, None], ax1[:, None], ax2[:, None]
# ), axis=1),
# scales=[scale0, scale1, scale2])
# o.add_object(obj_id=99, scene_obj=scene_obj,
# clone=False)
return o
def main(argv):
conf = Conf.get()
parser = argparse.ArgumentParser("Denis pose converter")
parser.add_argument('camera_name',
help="Camera name ('G15', 'S6')",
type=str)
parser.add_argument(
'-d',
dest='dir',
required=True,
help="Path to the <scene folder>/denis containing skeletons.json")
parser.add_argument(
'-filter',
dest='with_filtering',
action="store_true",
help="Should we do post-filtering (1-euro) on the pelvis positions")
parser.add_argument('-huber',
required=False,
help="Should we do huber loss?",
action='store_true')
parser.add_argument('-smooth',
type=float,
default=0.005,
help="Should we have a smoothness term (l2/huber)?")
parser.add_argument(
'--winsorize-limit',
type=float,
default=conf.optimize_path.winsorize_limit,
help='Threshold for filtering too large jumps of the 2D centroid')
parser.add_argument('--no-resample',
action='store_true',
help="add resampled frames")
parser.add_argument('--n-actors',
type=int,
default=1,
help="How many skeletons to track.")
parser.add_argument('-n-actors',
type=int,
default=1,
help="Max number of people in scene.")
# parser.add_argument(
# '-r', type=float,
# help='Video rate. Default: 1, if avconv -r 5. '
# 'Original video sampling rate (no subsampling) should be '
# '24/5=4.8. avconv -r 10 leads to 24/10=2.4.',
# required=True)
parser.add_argument('--person_height',
type=float,
help='Assumed height of human(s) in video.',
default=Conf.get().optimize_path.person_height)
parser.add_argument(
'--forwards-window-size',
type=int,
help='How many poses in time to look before AND after to '
'average forward direction. 0 means no averaging. Default: 0.',
default=0)
parser.add_argument('--no-img',
action='store_true',
help='Read and write images (vis reproj error)')
parser.add_argument('--postfix',
type=str,
help="output file postfix.",
default='unannot')
args = parser.parse_args(argv)
show = False
args.resample = not args.no_resample
# assert not args.resample, "resample should be off"
assert os.path.exists(args.dir), "Source does not exist: %s" % args.dir
p_scene = os.path.normpath(os.path.join(args.dir, os.pardir)) # type: str
p_video_params = os.path.join(p_scene, 'video_params.json')
assert os.path.exists(p_video_params), "Need video_params.json for rate"
if 'r' not in args or args.r is None:
args.r = json.load(open(p_video_params, 'r'))['rate-avconv']
# manual parameters (depth initialization, number of actors)
p_scene_params = os.path.join(args.dir, os.pardir, 'scene_params.json')
if not os.path.exists(p_scene_params):
scene_params = {
'depth_init': 10.,
'actors': args.n_actors,
'ground_rot': [0., 0., 0.]
}
json.dump(scene_params, open(p_scene_params, 'w'))
raise RuntimeError("Inited scene_params.json, please check: %s" %
p_scene_params)
else:
scene_params = json.load(open(p_scene_params, 'r'))
lg.warning("Will work with %d actors and init depth to %g" %
(scene_params['actors'], scene_params['depth_init']))
assert '--n-actors' not in argv \
or args.n_actors == scene_params['actors'], \
"Actor count mismatch, remove %d from args, because " \
"scene_params.json says %d?" \
% (args.n_actors, scene_params['actors'])
args.n_actors = scene_params['actors']
ground_rot = scene_params['ground_rot'] or [0., 0., 0.]
# load images
path_images = os.path.abspath(os.path.join(args.dir, os.pardir, 'origjpg'))
images = {}
shape_orig = None
if not args.no_img:
images, shape_orig = load_images(path_images)
path_skeleton = \
max((f for f in os.listdir(os.path.join(args.dir))
if f.startswith('skeletons') and f.endswith('json')),
key=lambda s: int(os.path.splitext(s)[0].split('_')[1]))
print("path_skeleton: %s" % path_skeleton)
data = json.load(open(os.path.join(args.dir, path_skeleton), 'r'))
# data, pose_constraints, first_run = \
# cleanup(data, p_dir=os.path.join(args.dir, os.pardir))
# poses_2d = []
# plt.figure()
# show_images(images, data)
if False:
# pose_ids = identify_actors_multi(data, n_actors=1)
p_segm_pickle = os.path.join(args.dir, os.pardir,
"label_skeletons.pickle")
problem = None
if False and os.path.exists(p_segm_pickle):
lg.warning("Loading skeleton segmentation from pickle %s" %
p_segm_pickle)
pose_ids, problem = pickle_load(open(p_segm_pickle, 'rb'))
if not problem or problem._n_actors != args.n_actors:
pose_ids, problem, data = more_actors_gurobi(
data,
n_actors=args.n_actors,
constraints=pose_constraints,
first_run=first_run)
if True or show:
show_multi(images,
data,
pose_ids,
problem,
p_dir=os.path.join(args.dir, os.pardir),
first_run=first_run,
n_actors=args.n_actors)
pickle.dump((pose_ids, problem), open(p_segm_pickle, 'wb'), -1)
else:
pose_ids = greedy_actors(data, n_actors=args.n_actors)
data = DataPosesWrapper(data=data)
visible_f = {a: {} for a in range(args.n_actors)}
visible_f_max = 0.
if show:
plt.ion()
fig = None
axe = None
scatters = dict()
# how many images we have
min_frame_id = min(f for f in pose_ids)
frames_mod = max(f for f in pose_ids) - min_frame_id + 1
skel_ours = Skeleton(frames_mod=frames_mod,
n_actors=args.n_actors,
min_frame_id=min_frame_id)
skel_ours_2d = Skeleton(frames_mod=frames_mod,
n_actors=args.n_actors,
min_frame_id=min_frame_id)
# assert len(images) == 0 or max(f for f in images) + 1 == frames_mod, \
# "Assumed image count is %d, but max_frame_id is %d" \
# % (len(images), frames_mod-1)
if isinstance(data, DataPosesWrapper):
frames = data.get_frames()
else:
frames = []
for frame_str in sorted(data.get_frames()):
try:
frame_id = int(frame_str.split('_')[1])
except ValueError:
print("skipping key %s" % frame_id)
continue
frames.append(frame_id)
my_visibilities = [[], []]
for frame_id in frames:
frame_str = DataPosesWrapper._to_frame_str(frame_id)
pose_in = data.get_poses_3d(frame_id=frame_id)
# np.asarray(data[frame_str][u'centered_3d'])
# pose_in_2d = np.asarray(data[frame_str][u'pose_2d'])
pose_in_2d = data.get_poses_2d(frame_id=frame_id)
# visible = np.asarray(data[frame_str][u'visible'])
if False and len(pose_in.shape) > 2:
pose_id = pose_ids[frame_id]
if not args.no_img:
im = cv2.cvtColor(images[frame_id], cv2.COLOR_RGB2BGR)
for i in range(pose_in.shape[0]):
c = (1., 0., 0., 1.)
if i == pose_id:
c = (0., 1., 0., 1.)
color = tuple(int(c_ * 255) for c_ in c[:3])
for p2d in pose_in_2d[i, :, :]:
# color = (c[0] * 255, c[1] * 255., c[2] * 255.)
cv2.circle(im, (p2d[1], p2d[0]),
radius=3,
color=color,
thickness=-1)
center = np.mean(pose_in_2d[i, :, :],
axis=0).round().astype('i4').tolist()
cv2.putText(im, "%d" % i, (center[1], center[0]), 1, 1,
color)
if show:
cv2.imshow("im", im)
cv2.waitKey(100)
# if sid not in scatters:
# scatters[sid] = axe.scatter(pose_in_2d[i, :, 1], pose_in_2d[i, :, 0], c=c)
# else:
# scatters[sid].set_offsets(pose_in_2d[i, :, [1, 0]])
# scatters[sid].set_array(np.tile(np.array(c), pose_in_2d.shape[1]))
# scatter.set_color(c)
# plt.draw()
# plt.pause(1.)
pose_in = pose_in[pose_id, :, :]
pose_in_2d = pose_in_2d[pose_id, :, :]
visible = visible[pose_id]
# else:
# pose_id = 0
# pose_id = pose_ids[frame_id]
for actor_id in range(args.n_actors):
# if actor_id in (2, 3, 4, 5, 8, 9)
# expanded frame_id
frame_id2 = Skeleton.unmod_frame_id(frame_id=frame_id,
actor_id=actor_id,
frames_mod=frames_mod)
assert (actor_id != 0) ^ (frame_id2 == frame_id), "no"
frame_id_mod = skel_ours.mod_frame_id(frame_id=frame_id2)
assert frame_id_mod == frame_id, \
"No: %d %d %d" % (frame_id, frame_id2, frame_id_mod)
actor_id2 = skel_ours.get_actor_id(frame_id2)
assert actor_id2 == actor_id, "no: %s %s" % (actor_id, actor_id2)
# which pose explains this actor in this frame
pose_id = pose_ids[frame_id][actor_id]
# check, if actor found
if pose_id < 0:
continue
# 3D pose
pose = pose_in[pose_id, :, JointDenis.revmap].T
# added by Aron on 4/4/2018 (Denis' pelvis is too high up)
pose[:, Joint.PELV] = (pose[:, Joint.LHIP] + pose[:, Joint.RHIP]) \
/ 2.
skel_ours.set_pose(frame_id2, pose)
# 2D pose
pose_2d = pose_in_2d[pose_id, :, :]
arr = np.array(JointDenis.pose_2d_to_ours(pose_2d),
dtype=np.float32).T
skel_ours_2d.set_pose(frame_id2, arr)
#
# visibility (binary) and confidence (float)
#
# np.asarray(data[frame_str][u'visible'][pose_id])
vis_i = data.get_visibilities(frame_id)[pose_id]
# vis_f = np.asarray(data[frame_str][u'visible_float'][pose_id])
vis_f = data.get_confidences(frame_id)[pose_id]
for jid, visible in enumerate(vis_i): # for each joint
# binary visibility
jid_ours = JointDenis.to_ours_2d(jid)
skel_ours_2d.set_visible(frame_id2, jid_ours, visible)
# confidence (fractional visibility)
if np.isnan(vis_f[jid]):
continue
try:
visible_f[actor_id][frame_id2][jid_ours] = vis_f[jid]
except KeyError:
visible_f[actor_id][frame_id2] = {jid_ours: vis_f[jid]}
visible_f_max = max(visible_f_max, vis_f[jid])
conf_ = get_conf_thresholded(vis_f[jid],
thresh_log_conf=None,
dtype_np=np.float32)
skel_ours_2d.set_confidence(frame_id=frame_id2,
joint=jid_ours,
confidence=conf_)
my_visibilities[0].append(vis_f[jid])
my_visibilities[1].append(conf_)
skel_ours_2d._confidence_normalized = True
plt.figure()
plt.plot(my_visibilities[0], my_visibilities[1], 'o')
plt.savefig('confidences.pdf')
assert skel_ours.n_actors == args.n_actors, "no"
assert skel_ours_2d.n_actors == args.n_actors, "no"
# align to room
min_z = np.min(skel_ours.poses[:, 2, :])
print("min_max: %s, %s" % (min_z, np.max(skel_ours.poses[:, 2, :])))
skel_ours.poses[:, 2, :] += min_z
skel_ours.poses /= 1000.
# The output is scaled to 2m by Denis.
# We change this to 1.8 * a scale in order to correct for
# the skeletons being a bit too high still.
skel_ours.poses *= \
args.person_height * conf.optimize_path.height_correction / 2.
skel_ours.poses[:, 2, :] *= -1.
skel_ours.poses = skel_ours.poses[:, [0, 2, 1], :]
# refine
name_video = args.dir.split(os.sep)[-2]
out_path = os.path.join(args.dir, os.pardir,
"skel_%s_%s.json" % (name_video, args.postfix))
out_path_orig = os.path.join(args.dir, os.pardir,
"skel_%s_lfd_orig.json" % name_video)
sclt_orig = Scenelet(skeleton=copy.deepcopy(skel_ours))
sclt_orig.save(out_path_orig)
skel_ours_2d_all = copy.deepcopy(skel_ours_2d)
assert len(skel_ours_2d_all.get_frames()), skel_ours_2d_all.get_frames()
#
# Optimize
#
# frames_ignore = [(282, 372), (516, 1000)]
skel_ours, skel_ours_2d, intrinsics, \
frame_ids_filled_in = prepare(
args.camera_name,
winsorize_limit=args.winsorize_limit,
shape_orig=shape_orig,
path_scene=p_scene,
skel_ours_2d=skel_ours_2d,
skel_ours=skel_ours,
resample=args.resample,
path_skel=path_skeleton)
frames_ignore = []
tr_ground = np.eye(4, dtype=np.float32)
skel_opt, out_images, K = \
optimize_path(
skel_ours, skel_ours_2d, images, intrinsics=intrinsics,
path_skel=out_path, shape_orig=shape_orig,
use_huber=args.huber, weight_smooth=args.smooth,
frames_ignore=frames_ignore, resample=args.resample,
depth_init=scene_params['depth_init'],
ground_rot=ground_rot)
for frame_id in skel_opt.get_frames():
skel_opt.set_time(frame_id=frame_id, time=float(frame_id) / args.r)
skel_opt_raw = copy.deepcopy(skel_opt)
skel_opt_resampled = Skeleton.resample(skel_opt)
# Filter pelvis
if args.with_filtering:
out_filter_path = os.path.join(args.dir, os.pardir, "vis_filtering")
skel_opt = filter_(skel_opt_resampled,
out_filter_path=out_filter_path,
skel_orig=skel_opt,
weight_smooth=args.smooth,
forwards_window_size=args.forwards_window_size)
else:
skel_opt.estimate_forwards(k=args.forwards_window_size)
skel_opt_resampled.estimate_forwards(k=args.forwards_window_size)
# if len(images):
# skel_opt.fill_with_closest(images.keys()[0], images.keys()[-1])
min_y, max_y = skel_opt.get_min_y(tr_ground)
print("min_y: %s, max_y: %s" % (min_y, max_y))
#
# save
#
frame_ids_old = set(skel_opt.get_frames())
if args.resample:
skel_opt = skel_opt_resampled
frame_ids_filled_in.update(
set(skel_opt.get_frames()).difference(frame_ids_old))
lg.warning("Saving resampled scenelet!")
scenelet = Scenelet(skel_opt)
del skel_opt
# skel_dict = skel_opt.to_json()
tr_ground[1, 3] = min_y
scenelet.aux_info['ground'] = tr_ground.tolist()
assert isinstance(ground_rot, list) and len(ground_rot) == 3
scenelet.add_aux_info('ground_rot', ground_rot)
scenelet.add_aux_info(
'path_opt_params', {
'rate': args.r,
'w-smooth': args.smooth,
'winsorize-limit': args.winsorize_limit,
'camera': args.camera_name,
'huber': args.huber,
'height_correction': conf.optimize_path.height_correction,
'focal_correction': conf.optimize_path.focal_correction
})
scenelet.add_aux_info('frame_ids_filled_in', list(frame_ids_filled_in))
# To MATLAB
# _skeleton.get_min_y(_tr_ground)
# with skel_opt as skeleton:
# skeleton = skel_opt
# skeleton_name = os.path.split(args.dir)[0]
# skeleton_name = skeleton_name[skeleton_name.rfind('/')+1:]
# mdict = skeleton.to_mdict(skeleton_name)
# mdict['room_transform'] = tr_ground
# mdict['room_transform'][1, 3] *= -1.
# print(mdict)
# print("scene_name?: %s" % os.path.split(args.dir)[0])
# skeleton.save_matlab(
# os.path.join(os.path.dirname(args.dir), "skeleton_opt.mat"),
# mdict=mdict)
assert scenelet.skeleton.has_forwards(), "No forwards??"
scenelet.save(out_path)
if show:
# save path plot
out_path_path = os.path.join(args.dir, os.pardir,
"%s_path.jpg" % name_video)
path_fig = plot_path(scenelet.skeleton)
legend = ["smooth %g" % args.smooth]
# hack debug
# path_skel2 = os.path.join(args.dir, os.pardir, 'skel_lobby7_nosmooth.json')
# if os.path.exists(path_skel2):
# skel2 = Skeleton.load(path_skel2)
# path_fig = plot_path(skel2, path_fig)
# legend.append('no smooth')
if show:
plt.legend(legend)
path_fig.savefig(out_path_path)
# backup args
path_args = os.path.join(args.dir, os.pardir, 'args_denis.txt')
with open(path_args, 'a') as f_args:
f_args.write("%s %s\n" %
(os.path.basename(sys.executable), " ".join(argv)))
# save 2D detections to file
if args.postfix == 'unannot':
path_skel_ours_2d = os.path.join(
args.dir, os.pardir, "skel_%s_2d_%02d.json" % (name_video, 0))
sclt_2d = Scenelet(skel_ours_2d_all)
print('Saving {} to {}'.format(len(skel_ours_2d_all.get_frames()),
path_skel_ours_2d))
sclt_2d.skeleton.aux_info = {}
sclt_2d.save(path_skel_ours_2d)
else:
print(args.postfix)
logging.info("Saving images...")
if len(images) and len(out_images):
path_out_images = os.path.join(args.dir, os.pardir, 'color')
try:
os.makedirs(path_out_images)
except OSError:
pass
visible_f_max_log = np.log(visible_f_max)
frames = list(out_images.keys())
for frame_id in range(frames[0], frames[-1] + 1):
im = out_images[frame_id] if frame_id in out_images \
else cv2.cvtColor(images[frame_id], cv2.COLOR_BGR2RGB)
for actor_id in range(args.n_actors):
if frame_id in visible_f[actor_id]:
frame_id2 = skel_ours_2d_all.unmod_frame_id(
frame_id=frame_id,
actor_id=actor_id,
frames_mod=skel_ours_2d_all.frames_mod)
for joint, is_vis in visible_f[actor_id][frame_id].items():
p2d = skel_ours_2d_all.get_joint_3d(joint,
frame_id=frame_id2)
# radius = np.log(is_vis) / visible_f_max_log
# lg.debug("r0: %g" % radius)
# radius = np.exp(np.log(is_vis) / visible_f_max_log)
# lg.debug("radius is %g" % radius)
vis_bool = True
if skel_ours_2d_all.has_visible(frame_id=frame_id2,
joint_id=joint):
vis_bool &= skel_ours_2d_all.is_visible(
frame_id2, joint)
radius = abs(np.log(is_vis / 0.1 + 1e-6))
if not np.isnan(radius):
p2d = (int(round(p2d[0])), int(round(p2d[1])))
cv2.circle(im,
center=p2d,
radius=int(round(radius)),
color=(1., 1., 1., 0.5),
thickness=1)
conf = get_conf_thresholded(conf=is_vis,
thresh_log_conf=None,
dtype_np=np.float32)
if conf > 0.5:
cv2.putText(img=im,
text=Joint(joint).get_name(),
org=p2d,
fontFace=1,
fontScale=1,
color=(10., 150., 10., 100.))
# lg.debug("set visibility to %g, radius %g" % (is_vis, radius))
# if frame_id in out_images:
scale = (shape_orig[1] / float(im.shape[1]),
shape_orig[0] / float(im.shape[0]))
cv2.imwrite(
os.path.join(path_out_images, "color_%05d.jpg" % frame_id),
cv2.resize(im, (0, 0),
fx=scale[0],
fy=scale[1],
interpolation=cv2.INTER_CUBIC))
# else:
# fname = "color_%05d.jpg" % frame_id
# shutil.copyfile(
# os.path.join(path_images, fname),
# os.path.join(path_out_images, fname))
lg.info("Wrote images to %s/" % path_out_images)
def prepare(camera_name, winsorize_limit, shape_orig, path_scene, skel_ours_2d,
skel_ours, resample, path_skel):
"""
Args:
camera_name (str):
Name of camera for intrinsics calculation.
winsorize_limit (float):
Outlier detection threshold.
shape_orig (Tuple[int, int]):
Original video resolution.
path_scene (str): Root path to scene.
skel_ours_2d (np.ndarray): (N, 2, 16)
2D skeletons from LFD in our format.
skel_ours (np.ndarray): (N, 3, 16)
Local space 3D skeletons in iMapper coordinate frame
(y-down, z-front).
resample (bool):
If needs densification using Blender's IK engine.
Returns:
skel_ours (Skeleton):
skel_ours_2d (Skeleton):
intrinsics (np.ndarray):
"""
assert camera_name is not None and isinstance(camera_name, str), \
"Need a camera name"
if shape_orig is None:
shape_orig = (np.float32(1080.), np.float32(1920.))
np.set_printoptions(linewidth=200, suppress=True)
if False:
plt.figure()
for i, frame_id in enumerate(skel_ours.get_frames()):
plot_2d(skel_ours_2d.get_pose(frame_id), images[frame_id])
plt.show()
path_intrinsics = os.path.join(path_scene, "intrinsics.json")
if os.path.exists(path_intrinsics):
lg.warning("Loading existing intrinsics matrix!")
K = np.array(json.load(open(path_intrinsics, 'r')), dtype=np.float32)
scale = (shape_orig[1] /
int(round(shape_orig[1] * float(INPUT_SIZE) / shape_orig[0])),
shape_orig[0] / float(INPUT_SIZE))
K[0, 0] /= scale[0]
K[0, 2] /= scale[0]
K[1, 1] /= scale[1]
K[1, 2] /= scale[1]
else:
K = intrinsics_matrix(INPUT_SIZE, shape_orig, camera_name)
focal_correction = Conf.get().optimize_path.focal_correction
if abs(focal_correction - 1.) > 1.e-3:
lg.warning("Warning, scaling intrinsics matrix by %f" %
focal_correction)
K[0, 0] *= focal_correction
K[1, 1] *= focal_correction
#print("K:\n%s,\nintr:\n%s" % (K, intr))
# sys.exit(0)
#
# Prune poses
#
skel_ours_2d, frame_ids_removed = filter_outliers(
skel_ours_2d, winsorize_limit=winsorize_limit, show=False)
frames_to_remove_3d = filter_wrong_poses(skel_ours_2d, skel_ours)
frames_to_ignore_list = set()
# if frames_ignore is not None:
# for start_end in frames_ignore:
# if isinstance(start_end, tuple):
# l_ = list(range(
# start_end[0],
# min(start_end[1], skel_ours_2d.get_frames()[-1])))
# frames_to_remove_3d.extend(l_)
# frames_to_ignore_list.update(l_)
# else:
# assert isinstance(start_end, int), \
# "Not int? %s" % repr(start_end)
# frames_to_remove_3d.append(start_end)
# frames_to_ignore_list.add(start_end)
for frame_id in skel_ours.get_frames():
if frame_id in frames_to_remove_3d:
skel_ours.remove_pose(frame_id)
# resample skeleton to fill in missing frames
skel_ours_old = skel_ours
frame_ids_filled_in = set(skel_ours_2d.get_frames()).difference(
set(skel_ours_old.get_frames()))
if resample:
lg.warning("Resampling BEFORE optimization")
# frames_to_resample = sorted(set(skel_ours_2d.get_frames()).difference(
# frames_to_ignore_list))
# skel_ours = Skeleton.resample(skel_ours_old,
# frame_ids=frames_to_resample)
# Aron on 6/4/2018
sclt_ours = Scenelet(skeleton=skel_ours)
stem = os.path.splitext(path_skel)[0]
path_filtered = "%s_filtered.json" % stem
path_ipoled = "%s_ikipol.json" % os.path.splitext(path_filtered)[0]
if not os.path.exists(path_ipoled):
sclt_ours.save(path_filtered)
script_filepath = \
os.path.normpath(os.path.join(
os.path.dirname(os.path.abspath(__file__)),
os.pardir, 'blender', 'ipol_ik.py'))
assert os.path.exists(script_filepath), "No: %s" % script_filepath
blender_path = os.environ.get('BLENDER')
if not os.path.isfile(blender_path):
raise RuntimeError(
"Need \"BLENDER\" environment variable to be set "
"to the blender executable")
cmd_params = [
blender_path, '-noaudio', '-b', '-P', script_filepath, '--',
path_filtered
]
print("calling %s" % " ".join(cmd_params))
ret = check_call(cmd_params)
print("ret: %s" % ret)
else:
lg.warning("\n\n\tNOT recomputing IK interpolation, "
"file found at %s!\n" % path_ipoled)
skel_ours = Scenelet.load(path_ipoled, no_obj=True).skeleton
# remove extra frames at ends and beginnings of actors
spans = skel_ours_old.get_actor_empty_frames()
old_frames = skel_ours_old.get_frames()
frames_to_remove = []
for frame_id in skel_ours.get_frames():
if frame_id not in old_frames:
in_spans = next(
(True for span in spans if span[0] < frame_id < span[1]),
None)
if in_spans:
frames_to_remove.append(frame_id)
# lg.debug("diff: %s (a%s, f%s)"
# % (
# frame_id,
# skel_ours_old.get_actor_id(frame_id),
# skel_ours_old.mod_frame_id(frame_id)
# ))
for frame_id in frames_to_remove:
skel_ours.remove_pose(frame_id)
for frame_id in skel_ours_2d.get_frames():
if not skel_ours.has_pose(frame_id):
skel_ours_2d.remove_pose(frame_id)
for frame_id in skel_ours.get_frames():
if not skel_ours_2d.has_pose(frame_id):
skel_ours.remove_pose(frame_id)
frames_set_ours = set(skel_ours.get_frames())
frames_set_2d = set(skel_ours_2d.get_frames())
if frames_set_ours != frames_set_2d:
print("Frame mismatch: %s" % frames_set_ours.difference(frames_set_2d))
lg.warning("Removing pelvis and neck from 2D input")
for frame_id in skel_ours_2d.get_frames():
skel_ours_2d.set_visible(frame_id, Joint.PELV, 0)
skel_ours_2d.set_visible(frame_id, Joint.NECK, 0)
return skel_ours, skel_ours_2d, K, frame_ids_filled_in