def select_gaps(argv):
pjoin = os.path.join
parser = argparse.ArgumentParser("Select gaps")
parser = parser_video(parser)
parser.add_argument('--select-gaps', action='store_true')
parser.add_argument("-s",
"--d-scenelets",
dest='s',
type=argparse_check_exists,
help="Folder containing original PiGraphs scenelets")
args = parser.parse_args(argv)
d = os.path.join(args.video, args.d)
assert os.path.exists(d), "does not exist: %s" % d
if os.path.isdir(args.video):
if args.video.endswith(os.sep):
args.video = args.video[:-1]
name_query = args.video.split(os.sep)[-1]
assert len(name_query), args.video.split(os.sep)
p_query = pjoin(args.video, "skel_%s_unannot.json" % name_query) \
if os.path.isdir(args.video) else args.video
assert p_query.endswith('.json'), "Need a skeleton file"
# load initial video path (local poses)
query = Scenelet.load(p_query, no_obj=True)
frame_ids = query.skeleton.get_frames()
centroids = Skeleton.get_resampled_centroids(start=frame_ids[0],
end=frame_ids[-1],
old_frame_ids=frame_ids,
poses=query.skeleton.poses)
depths_times_charnesses = []
skeleton = Skeleton()
depths = []
for p in sorted(os.listdir(d)):
ch = float(p.split('charness')[1][1:])
d_time = pjoin(d, p)
p_skel = next(
f for f in os.listdir(d_time) if os.path.isfile(pjoin(d_time, f))
and f.startswith('skel') and f.endswith('json') and '_00' in f)
sclt = Scenelet.load(pjoin(d_time, p_skel))
mn, mx = sclt.skeleton.get_frames_min_max()
frame_id = mn + (mx - mn) // 2
if query.skeleton.has_pose(frame_id):
pos_3d = query.skeleton.get_centroid_3d(frame_id)
else:
lin_id = frame_id - frame_ids[0]
pos_3d = centroids[lin_id, :]
depth = np.linalg.norm(pos_3d)
depths.append(depth)
depths_times_charnesses.append(
DepthTimeCharness(depth=depth, frame_id=frame_id, charness=ch))
hist, bin_edges = np.histogram(depths, bins=5)
lg.debug("hist: %s" % hist)
lg.debug("edges: %s" % bin_edges)
def __init__(self, skeleton=None):
self.skeleton = skeleton if skeleton is not None else Skeleton()
self.objects = {}
self.aux_info = {}
self.transforms = {}
self.name_scene = None
self.name_scenelet = None
self.charness = np.float32(1.)
"""Per-scenelet charness (a single float32)."""
self.match_score = np.float32(0.)
"""Also called feature distance, smaller is better."""
# self.confidence = None
# """Array to show how sure we are about the different
# entries in the skeleton."""
self.density = None
"""How densely sampled in space this part of the scene is
(how many poses there are usually around the middle frame).
"""
self.charness_poses = None
"""A dictionary that stores a characteristicness for each pose.
def extract_skeleton(scene,
frame_ids=None,
frame_multiplier=1.,
time_multiplier=1.):
joints = {
ob.name.split('.')[1]: ob
for ob in bpy.data.objects
if ob.name.startswith('Output') and ob.name.endswith('Sphere')
}
print("joints: %s" % joints)
assert len(joints) == Joint.get_num_joints(), \
"No: %s != %s" % (len(joints), Joint.get_num_joints())
if not frame_ids:
frame_ids = range(scene.frame_start, scene.frame_end + 1)
skeleton = Skeleton()
for frame_id in frame_ids:
o_frame_id = int(round(frame_id * frame_multiplier))
if skeleton.has_pose(o_frame_id):
print("skipping %s, because already set" % frame_id)
continue
print("frame_id: %s, o_frame_id: %s" % (frame_id, o_frame_id))
scene.frame_set(frame_id)
bpy.context.scene.update()
pose = np.zeros(shape=(Skeleton.DIM, len(joints)))
for joint, ob in joints.items():
pos = ob.matrix_world.col[3]
joint_id = Joint.from_string(joint)
pose[:, joint_id] = from_blender(pos)
assert not skeleton.has_pose(o_frame_id), "Already has %s" % frame_id
skeleton.set_pose(frame_id=o_frame_id,
pose=pose,
time=o_frame_id * time_multiplier)
# scenelet = Scenelet(skeleton=skeleton)
# scenelet.save(path=path_out, save_obj=False)
return skeleton
def extract_annotated_scenelet(
scene,
prefix_obj='obb',
frame_ids=None,
frame_multiplier=1.,
time_multiplier=1.,
f_ob_is_joint=lambda ob: ob.name.startswith(
'Output') and ob.name.endswith('Sphere'),
f_joint_name_from_ob=lambda ob: ob.name.split('.')[1]):
"""
Args:
scene (bpy.types.Scene):
The current scene (e.g. bpy.context.scene).
prefix_obj (str):
Start of object names that we want to include in the scenelet
as oriented bounding boxes.
frame_ids (List[int]):
A subset of frame IDs to export.
frame_multiplier (float):
Scaling for frame IDs. The result will be rounded and truncated.
output.frame_id := int(round(frame_id * frame_multiplier))
time_multipler (float):
Scaling for times associated with frame_ids.
output.time := int(round(frame_id * frame_multiplier))
* time_multiplier.
f_ob_is_joint (Callable[[bpy.types.Object], bool]]):
Decides if a Blender object is a joint.
f_joint_name_from_ob (Callable[[bpy.types.Object], str]):
Gets the joint name from the Blender object name.
"""
# joints = {
# ob.name.split('.')[1]: ob
# for ob in bpy.data.objects
# if ob.name.startswith('Output') and ob.name.endswith('Sphere')}
joints = {
f_joint_name_from_ob(ob): ob
for ob in bpy.data.objects if f_ob_is_joint(ob)
}
print("joints: %s" % joints)
skeleton = Skeleton()
if len(joints):
assert len(joints) == 16, "No: %s" % len(joints)
if not frame_ids:
frame_ids = range(scene.frame_start, scene.frame_end + 1)
for frame_id in frame_ids:
o_frame_id = int(round(frame_id * frame_multiplier))
if skeleton.has_pose(o_frame_id):
print("skipping %s" % frame_id)
continue
print("frame_id: %s" % frame_id)
scene.frame_set(frame_id)
bpy.context.scene.update()
# bpy.ops.anim.change_frame(frame_id)
pose = np.zeros(shape=(3, len(joints)))
for joint, ob in joints.items():
pos = ob.matrix_world.col[3]
print("pos[%s]: %s" % (ob.name, pos))
joint_id = Joint.from_string(joint)
print("joint %s is %s" % (joint, Joint(joint_id)))
pose[:, joint_id] = from_blender(pos)
print("o_frame: %s from %s" % (o_frame_id, frame_id))
assert not skeleton.has_pose(o_frame_id), \
"Already has %s" % frame_id
skeleton.set_pose(frame_id=o_frame_id,
pose=pose,
time=o_frame_id * time_multiplier)
objs_bl = {}
for obj in bpy.data.objects:
if obj.name.startswith(prefix_obj) and not obj.hide:
obj_id = int(obj.name.split('_')[1])
try:
objs_bl[obj_id].append(obj)
except KeyError:
objs_bl[obj_id] = [obj]
print("objs: %s" % objs_bl)
scenelet = Scenelet(skeleton=skeleton)
print("scenelet: %s" % scenelet)
for obj_id, parts_bl in objs_bl.items():
name_category = None
scene_obj = None
for part_id, part_bl in enumerate(parts_bl):
transl, rot, scale = part_bl.matrix_world.decompose()
rot = rot.to_matrix()
if any(comp < 0. for comp in scale):
scale *= -1.
rot *= -1.
assert not any(comp < 0. for comp in scale), "No: %s" % scale
matrix_world = part_bl.matrix_world.copy()
# need to save full scale, not only half axes
for c in range(3):
for r in range(3):
matrix_world[r][c] *= 2.
name_parts = part_bl.name.split('_')
if name_category is None:
name_category = name_parts[2]
scene_obj = SceneObj(label=name_category)
else:
assert name_category == name_parts[2], \
"No: %s %s" % (name_category, name_parts[2])
name_part = name_parts[3]
print("part: %s" % name_part)
part = SceneObjPart(name_part)
part.obb = Obb(centroid=np.array(
from_blender([transl[0], transl[1], transl[2]])),
axes=np.array([[rot[0][0], rot[0][1], rot[0][2]],
[-rot[2][0], -rot[2][1], -rot[2][2]],
[rot[1][0], rot[1][1], rot[1][2]]]),
scales=np.array(
[scale[0] * 2., scale[1] * 2., scale[2] * 2.]))
# if 'table' in name_category:
# print(part.obb.axes)
# raise RuntimeError("stop")
print("obb: %s" % part.obb.to_json(0))
scene_obj.add_part(part_id, part)
scenelet.add_object(obj_id, scene_obj, clone=False)
return scenelet
def load(cls, path, no_obj=False):
"""
Returns:
s (stealth.logic.scenelet.Scenelet):
Scenelet read from disk.
"""
with open(path, 'r') as fin:
data = json.load(fin)
s = Scenelet()
# 3d
if '3d' in data:
s.skeleton = Skeleton.from_json(data)
else:
sys.stderr.write("No skeleton in scenelet: %s" % path)
# scenelets
if 'scenelets' in data:
if 'obbs' in data['scenelets'] and not no_obj:
for obb_path in data['scenelets']['obbs']:
obb_folder, obj_name = os.path.split(obb_path)
obj_id, scene_obj_name, part_name = obj_name.split('_')
obj_id = int(obj_id)
part_name = part_name.split('.')[0]
obb_json_path = os.path.join(
os.path.split(path)[0], obb_folder,
os.path.splitext(obj_name)[0] + ".json")
if os.path.exists(obb_json_path):
part = SceneObjPart(part_name)
part.obb, data_obb = Obb.load(obb_json_path,
return_data=True)
part_id = data_obb['part_id'] \
if 'part_id' in data_obb \
else -1
part.path = os.path.join(
os.path.split(path)[0], obb_path)
if s.has_object(obj_id):
s.get_object(obj_id).add_part(part_id=part_id,
label_or_part=part)
else:
scene_obj = SceneObj(scene_obj_name)
scene_obj.add_part(part_id=part_id,
label_or_part=part)
s.add_object(obj_id, scene_obj, clone=False)
else:
obb_obj_path = os.path.join(
os.path.split(path)[0], obb_path)
s.add_object(obj_id,
SceneObject(label=scene_obj_name,
name=part_name,
mesh=MeshOBJ(obb_obj_path,
verbose=False)),
clone=False)
if 'transforms' in data['scenelets']:
try:
s.objects[obj_id].transform = \
np.asarray(
data['scenelets']['transforms'][obj_name],
dtype='f4')
except KeyError:
pass
# TODO: read 'clouds' to parts
for k, v in data.items():
if k in Scenelet.__KEYS_SKELETON:
continue
s.add_aux_info(k, v)
try:
s.name_scene = data['name_scene']
except KeyError:
s.name_scene = os.path.basename(os.path.split(path)[-2])
if '__' in s.name_scene:
s.name_scene = s.name_scene.partition('__')[0]
if s.name_scene.startswith('skel_'):
s.name_scene = s.name_scene[5:]
try:
s.name_scenelet = data['name_scenelet']
except KeyError:
s.name_scenelet = os.path.splitext(os.path.basename(path))[0]
if s.name_scenelet.startswith('skel_'):
s.name_scenelet = s.name_scenelet[5:]
if 'charness' in data:
s.charness = data['charness']
if 'match_score' in data:
s.match_score = data['match_score']
# if 'confidence' in data:
# s.confidence = data['confidence']
# density
try:
s.density = data['density']
except KeyError:
pass
# charness_poses
try:
j_dict = data['charness_poses']
s.charness_poses = {
frame_id: np.float32(value)
for frame_id, value in zip(j_dict['frame_ids'],
j_dict['values'])
}
except KeyError:
pass
return s
def main(argv):
parser = argparse.ArgumentParser(
"Filter initial path based on distance to full fit")
parser.add_argument('skel', help="Skeleton file to filter", type=str)
parser.add_argument('--threshold',
help='Distance threshold. Default: 0.4',
type=float,
default=0.4)
args = parser.parse_args(argv)
lower_body = [
Joint.LKNE, Joint.RKNE, Joint.LANK, Joint.RANK, Joint.LHIP, Joint.RHIP
]
print(args.skel)
p_root = os.path.dirname(args.skel)
p_fit = os.path.join(p_root, 'opt1')
assert os.path.isdir(p_fit), p_fit
query = Scenelet.load(args.skel)
out = Skeleton()
data = []
x = []
y = []
y2 = []
for d_ in sorted(os.listdir(p_fit)):
d = os.path.join(p_fit, d_)
pattern = os.path.join(d, 'skel_*.json')
for f in sorted(glob.iglob(pattern)):
print(f)
assert '00' in f, f
sclt = Scenelet.load(f)
frames = sclt.skeleton.get_frames()
mid_frame = frames[len(frames) // 2]
time = sclt.skeleton.get_time(mid_frame)
q_frame_id = query.skeleton.find_time(time)
q_time = query.skeleton.get_time(q_frame_id)
print(time, q_time, f)
q_pose = query.skeleton.get_pose(q_frame_id)
pose = sclt.skeleton.get_pose(mid_frame)
pose[[0, 2]] -= (pose[:, Joint.PELV:Joint.PELV + 1] -
q_pose[:, Joint.PELV:Joint.PELV + 1])[[0, 2]]
diff = np.mean(
np.linalg.norm(q_pose[:, lower_body] - pose[:, lower_body],
axis=0))
print(q_frame_id, time, diff)
y.append(diff)
x.append(q_frame_id)
data.append((q_frame_id, diff, time))
if query.skeleton.has_pose(q_frame_id - 1):
tmp_pose = copy.deepcopy(q_pose)
tmp_pose -= tmp_pose[:, Joint.PELV:Joint.PELV +
1] - query.skeleton.get_pose(
q_frame_id -
1)[:, Joint.PELV:Joint.PELV + 1]
y2.append(
np.mean(
np.linalg.norm(pose[:, lower_body] -
tmp_pose[:, lower_body],
axis=0)))
else:
y2.append(0.)
out.set_pose(frame_id=q_frame_id, time=q_time, pose=pose)
break
data = smooth(data)
plt.plot(x, y, 'x--', label='Distance to best Kinect fit\'s center frame')
plt.plot(x, y2, 'o--', label='Distance to prev pose')
plt.plot([d[0] for d in data], [d[1] for d in data],
'o--',
label='Smoothed')
plt.xlabel('Time (s)')
plt.ylabel('Sum local squared distance')
plt.legend()
plt.savefig(os.path.join(p_root, 'tmp.pdf'))
Scenelet(skeleton=out).save(os.path.join(p_root, 'skel_tmp.json'))
above = []
prev_frame_id = None
for frame_id, dist, time in data:
# assert prev_frame_id is None or frame_id != prev_frame_id, \
# 'No: {}'.format(frame_id)
if dist > args.threshold:
above.append(
Span2(start=frame_id, end=frame_id, value=dist, time=time))
prev_frame_id = frame_id
spans = [copy.deepcopy(above[0])]
it = iter(above)
next(it)
prev_frame_id = above[0].start
for span2 in it:
frame_id = span2.start
if prev_frame_id + 1 < frame_id:
# span = spans[-1]
# spans[-1] = span[0], prev_frame_id, span[2]
spans[-1].end = prev_frame_id
spans.append(
Span2(start=frame_id,
end=frame_id,
time=None,
value=span2.value))
else:
print(prev_frame_id, frame_id)
prev_frame_id = frame_id
spans[-1].end = prev_frame_id
print("Need replacement: {}".format(above))
print("Need replacement2: {}".format(spans))
def main(argv):
conf = Conf.get()
parser = argparse.ArgumentParser("Denis pose converter")
parser.add_argument('camera_name',
help="Camera name ('G15', 'S6')",
type=str)
parser.add_argument(
'-d',
dest='dir',
required=True,
help="Path to the <scene folder>/denis containing skeletons.json")
parser.add_argument(
'-filter',
dest='with_filtering',
action="store_true",
help="Should we do post-filtering (1-euro) on the pelvis positions")
parser.add_argument('-huber',
required=False,
help="Should we do huber loss?",
action='store_true')
parser.add_argument('-smooth',
type=float,
default=0.005,
help="Should we have a smoothness term (l2/huber)?")
parser.add_argument(
'--winsorize-limit',
type=float,
default=conf.optimize_path.winsorize_limit,
help='Threshold for filtering too large jumps of the 2D centroid')
parser.add_argument('--no-resample',
action='store_true',
help="add resampled frames")
parser.add_argument('--n-actors',
type=int,
default=1,
help="How many skeletons to track.")
parser.add_argument('-n-actors',
type=int,
default=1,
help="Max number of people in scene.")
# parser.add_argument(
# '-r', type=float,
# help='Video rate. Default: 1, if avconv -r 5. '
# 'Original video sampling rate (no subsampling) should be '
# '24/5=4.8. avconv -r 10 leads to 24/10=2.4.',
# required=True)
parser.add_argument('--person_height',
type=float,
help='Assumed height of human(s) in video.',
default=Conf.get().optimize_path.person_height)
parser.add_argument(
'--forwards-window-size',
type=int,
help='How many poses in time to look before AND after to '
'average forward direction. 0 means no averaging. Default: 0.',
default=0)
parser.add_argument('--no-img',
action='store_true',
help='Read and write images (vis reproj error)')
parser.add_argument('--postfix',
type=str,
help="output file postfix.",
default='unannot')
args = parser.parse_args(argv)
show = False
args.resample = not args.no_resample
# assert not args.resample, "resample should be off"
assert os.path.exists(args.dir), "Source does not exist: %s" % args.dir
p_scene = os.path.normpath(os.path.join(args.dir, os.pardir)) # type: str
p_video_params = os.path.join(p_scene, 'video_params.json')
assert os.path.exists(p_video_params), "Need video_params.json for rate"
if 'r' not in args or args.r is None:
args.r = json.load(open(p_video_params, 'r'))['rate-avconv']
# manual parameters (depth initialization, number of actors)
p_scene_params = os.path.join(args.dir, os.pardir, 'scene_params.json')
if not os.path.exists(p_scene_params):
scene_params = {
'depth_init': 10.,
'actors': args.n_actors,
'ground_rot': [0., 0., 0.]
}
json.dump(scene_params, open(p_scene_params, 'w'))
raise RuntimeError("Inited scene_params.json, please check: %s" %
p_scene_params)
else:
scene_params = json.load(open(p_scene_params, 'r'))
lg.warning("Will work with %d actors and init depth to %g" %
(scene_params['actors'], scene_params['depth_init']))
assert '--n-actors' not in argv \
or args.n_actors == scene_params['actors'], \
"Actor count mismatch, remove %d from args, because " \
"scene_params.json says %d?" \
% (args.n_actors, scene_params['actors'])
args.n_actors = scene_params['actors']
ground_rot = scene_params['ground_rot'] or [0., 0., 0.]
# load images
path_images = os.path.abspath(os.path.join(args.dir, os.pardir, 'origjpg'))
images = {}
shape_orig = None
if not args.no_img:
images, shape_orig = load_images(path_images)
path_skeleton = \
max((f for f in os.listdir(os.path.join(args.dir))
if f.startswith('skeletons') and f.endswith('json')),
key=lambda s: int(os.path.splitext(s)[0].split('_')[1]))
print("path_skeleton: %s" % path_skeleton)
data = json.load(open(os.path.join(args.dir, path_skeleton), 'r'))
# data, pose_constraints, first_run = \
# cleanup(data, p_dir=os.path.join(args.dir, os.pardir))
# poses_2d = []
# plt.figure()
# show_images(images, data)
if False:
# pose_ids = identify_actors_multi(data, n_actors=1)
p_segm_pickle = os.path.join(args.dir, os.pardir,
"label_skeletons.pickle")
problem = None
if False and os.path.exists(p_segm_pickle):
lg.warning("Loading skeleton segmentation from pickle %s" %
p_segm_pickle)
pose_ids, problem = pickle_load(open(p_segm_pickle, 'rb'))
if not problem or problem._n_actors != args.n_actors:
pose_ids, problem, data = more_actors_gurobi(
data,
n_actors=args.n_actors,
constraints=pose_constraints,
first_run=first_run)
if True or show:
show_multi(images,
data,
pose_ids,
problem,
p_dir=os.path.join(args.dir, os.pardir),
first_run=first_run,
n_actors=args.n_actors)
pickle.dump((pose_ids, problem), open(p_segm_pickle, 'wb'), -1)
else:
pose_ids = greedy_actors(data, n_actors=args.n_actors)
data = DataPosesWrapper(data=data)
visible_f = {a: {} for a in range(args.n_actors)}
visible_f_max = 0.
if show:
plt.ion()
fig = None
axe = None
scatters = dict()
# how many images we have
min_frame_id = min(f for f in pose_ids)
frames_mod = max(f for f in pose_ids) - min_frame_id + 1
skel_ours = Skeleton(frames_mod=frames_mod,
n_actors=args.n_actors,
min_frame_id=min_frame_id)
skel_ours_2d = Skeleton(frames_mod=frames_mod,
n_actors=args.n_actors,
min_frame_id=min_frame_id)
# assert len(images) == 0 or max(f for f in images) + 1 == frames_mod, \
# "Assumed image count is %d, but max_frame_id is %d" \
# % (len(images), frames_mod-1)
if isinstance(data, DataPosesWrapper):
frames = data.get_frames()
else:
frames = []
for frame_str in sorted(data.get_frames()):
try:
frame_id = int(frame_str.split('_')[1])
except ValueError:
print("skipping key %s" % frame_id)
continue
frames.append(frame_id)
my_visibilities = [[], []]
for frame_id in frames:
frame_str = DataPosesWrapper._to_frame_str(frame_id)
pose_in = data.get_poses_3d(frame_id=frame_id)
# np.asarray(data[frame_str][u'centered_3d'])
# pose_in_2d = np.asarray(data[frame_str][u'pose_2d'])
pose_in_2d = data.get_poses_2d(frame_id=frame_id)
# visible = np.asarray(data[frame_str][u'visible'])
if False and len(pose_in.shape) > 2:
pose_id = pose_ids[frame_id]
if not args.no_img:
im = cv2.cvtColor(images[frame_id], cv2.COLOR_RGB2BGR)
for i in range(pose_in.shape[0]):
c = (1., 0., 0., 1.)
if i == pose_id:
c = (0., 1., 0., 1.)
color = tuple(int(c_ * 255) for c_ in c[:3])
for p2d in pose_in_2d[i, :, :]:
# color = (c[0] * 255, c[1] * 255., c[2] * 255.)
cv2.circle(im, (p2d[1], p2d[0]),
radius=3,
color=color,
thickness=-1)
center = np.mean(pose_in_2d[i, :, :],
axis=0).round().astype('i4').tolist()
cv2.putText(im, "%d" % i, (center[1], center[0]), 1, 1,
color)
if show:
cv2.imshow("im", im)
cv2.waitKey(100)
# if sid not in scatters:
# scatters[sid] = axe.scatter(pose_in_2d[i, :, 1], pose_in_2d[i, :, 0], c=c)
# else:
# scatters[sid].set_offsets(pose_in_2d[i, :, [1, 0]])
# scatters[sid].set_array(np.tile(np.array(c), pose_in_2d.shape[1]))
# scatter.set_color(c)
# plt.draw()
# plt.pause(1.)
pose_in = pose_in[pose_id, :, :]
pose_in_2d = pose_in_2d[pose_id, :, :]
visible = visible[pose_id]
# else:
# pose_id = 0
# pose_id = pose_ids[frame_id]
for actor_id in range(args.n_actors):
# if actor_id in (2, 3, 4, 5, 8, 9)
# expanded frame_id
frame_id2 = Skeleton.unmod_frame_id(frame_id=frame_id,
actor_id=actor_id,
frames_mod=frames_mod)
assert (actor_id != 0) ^ (frame_id2 == frame_id), "no"
frame_id_mod = skel_ours.mod_frame_id(frame_id=frame_id2)
assert frame_id_mod == frame_id, \
"No: %d %d %d" % (frame_id, frame_id2, frame_id_mod)
actor_id2 = skel_ours.get_actor_id(frame_id2)
assert actor_id2 == actor_id, "no: %s %s" % (actor_id, actor_id2)
# which pose explains this actor in this frame
pose_id = pose_ids[frame_id][actor_id]
# check, if actor found
if pose_id < 0:
continue
# 3D pose
pose = pose_in[pose_id, :, JointDenis.revmap].T
# added by Aron on 4/4/2018 (Denis' pelvis is too high up)
pose[:, Joint.PELV] = (pose[:, Joint.LHIP] + pose[:, Joint.RHIP]) \
/ 2.
skel_ours.set_pose(frame_id2, pose)
# 2D pose
pose_2d = pose_in_2d[pose_id, :, :]
arr = np.array(JointDenis.pose_2d_to_ours(pose_2d),
dtype=np.float32).T
skel_ours_2d.set_pose(frame_id2, arr)
#
# visibility (binary) and confidence (float)
#
# np.asarray(data[frame_str][u'visible'][pose_id])
vis_i = data.get_visibilities(frame_id)[pose_id]
# vis_f = np.asarray(data[frame_str][u'visible_float'][pose_id])
vis_f = data.get_confidences(frame_id)[pose_id]
for jid, visible in enumerate(vis_i): # for each joint
# binary visibility
jid_ours = JointDenis.to_ours_2d(jid)
skel_ours_2d.set_visible(frame_id2, jid_ours, visible)
# confidence (fractional visibility)
if np.isnan(vis_f[jid]):
continue
try:
visible_f[actor_id][frame_id2][jid_ours] = vis_f[jid]
except KeyError:
visible_f[actor_id][frame_id2] = {jid_ours: vis_f[jid]}
visible_f_max = max(visible_f_max, vis_f[jid])
conf_ = get_conf_thresholded(vis_f[jid],
thresh_log_conf=None,
dtype_np=np.float32)
skel_ours_2d.set_confidence(frame_id=frame_id2,
joint=jid_ours,
confidence=conf_)
my_visibilities[0].append(vis_f[jid])
my_visibilities[1].append(conf_)
skel_ours_2d._confidence_normalized = True
plt.figure()
plt.plot(my_visibilities[0], my_visibilities[1], 'o')
plt.savefig('confidences.pdf')
assert skel_ours.n_actors == args.n_actors, "no"
assert skel_ours_2d.n_actors == args.n_actors, "no"
# align to room
min_z = np.min(skel_ours.poses[:, 2, :])
print("min_max: %s, %s" % (min_z, np.max(skel_ours.poses[:, 2, :])))
skel_ours.poses[:, 2, :] += min_z
skel_ours.poses /= 1000.
# The output is scaled to 2m by Denis.
# We change this to 1.8 * a scale in order to correct for
# the skeletons being a bit too high still.
skel_ours.poses *= \
args.person_height * conf.optimize_path.height_correction / 2.
skel_ours.poses[:, 2, :] *= -1.
skel_ours.poses = skel_ours.poses[:, [0, 2, 1], :]
# refine
name_video = args.dir.split(os.sep)[-2]
out_path = os.path.join(args.dir, os.pardir,
"skel_%s_%s.json" % (name_video, args.postfix))
out_path_orig = os.path.join(args.dir, os.pardir,
"skel_%s_lfd_orig.json" % name_video)
sclt_orig = Scenelet(skeleton=copy.deepcopy(skel_ours))
sclt_orig.save(out_path_orig)
skel_ours_2d_all = copy.deepcopy(skel_ours_2d)
assert len(skel_ours_2d_all.get_frames()), skel_ours_2d_all.get_frames()
#
# Optimize
#
# frames_ignore = [(282, 372), (516, 1000)]
skel_ours, skel_ours_2d, intrinsics, \
frame_ids_filled_in = prepare(
args.camera_name,
winsorize_limit=args.winsorize_limit,
shape_orig=shape_orig,
path_scene=p_scene,
skel_ours_2d=skel_ours_2d,
skel_ours=skel_ours,
resample=args.resample,
path_skel=path_skeleton)
frames_ignore = []
tr_ground = np.eye(4, dtype=np.float32)
skel_opt, out_images, K = \
optimize_path(
skel_ours, skel_ours_2d, images, intrinsics=intrinsics,
path_skel=out_path, shape_orig=shape_orig,
use_huber=args.huber, weight_smooth=args.smooth,
frames_ignore=frames_ignore, resample=args.resample,
depth_init=scene_params['depth_init'],
ground_rot=ground_rot)
for frame_id in skel_opt.get_frames():
skel_opt.set_time(frame_id=frame_id, time=float(frame_id) / args.r)
skel_opt_raw = copy.deepcopy(skel_opt)
skel_opt_resampled = Skeleton.resample(skel_opt)
# Filter pelvis
if args.with_filtering:
out_filter_path = os.path.join(args.dir, os.pardir, "vis_filtering")
skel_opt = filter_(skel_opt_resampled,
out_filter_path=out_filter_path,
skel_orig=skel_opt,
weight_smooth=args.smooth,
forwards_window_size=args.forwards_window_size)
else:
skel_opt.estimate_forwards(k=args.forwards_window_size)
skel_opt_resampled.estimate_forwards(k=args.forwards_window_size)
# if len(images):
# skel_opt.fill_with_closest(images.keys()[0], images.keys()[-1])
min_y, max_y = skel_opt.get_min_y(tr_ground)
print("min_y: %s, max_y: %s" % (min_y, max_y))
#
# save
#
frame_ids_old = set(skel_opt.get_frames())
if args.resample:
skel_opt = skel_opt_resampled
frame_ids_filled_in.update(
set(skel_opt.get_frames()).difference(frame_ids_old))
lg.warning("Saving resampled scenelet!")
scenelet = Scenelet(skel_opt)
del skel_opt
# skel_dict = skel_opt.to_json()
tr_ground[1, 3] = min_y
scenelet.aux_info['ground'] = tr_ground.tolist()
assert isinstance(ground_rot, list) and len(ground_rot) == 3
scenelet.add_aux_info('ground_rot', ground_rot)
scenelet.add_aux_info(
'path_opt_params', {
'rate': args.r,
'w-smooth': args.smooth,
'winsorize-limit': args.winsorize_limit,
'camera': args.camera_name,
'huber': args.huber,
'height_correction': conf.optimize_path.height_correction,
'focal_correction': conf.optimize_path.focal_correction
})
scenelet.add_aux_info('frame_ids_filled_in', list(frame_ids_filled_in))
# To MATLAB
# _skeleton.get_min_y(_tr_ground)
# with skel_opt as skeleton:
# skeleton = skel_opt
# skeleton_name = os.path.split(args.dir)[0]
# skeleton_name = skeleton_name[skeleton_name.rfind('/')+1:]
# mdict = skeleton.to_mdict(skeleton_name)
# mdict['room_transform'] = tr_ground
# mdict['room_transform'][1, 3] *= -1.
# print(mdict)
# print("scene_name?: %s" % os.path.split(args.dir)[0])
# skeleton.save_matlab(
# os.path.join(os.path.dirname(args.dir), "skeleton_opt.mat"),
# mdict=mdict)
assert scenelet.skeleton.has_forwards(), "No forwards??"
scenelet.save(out_path)
if show:
# save path plot
out_path_path = os.path.join(args.dir, os.pardir,
"%s_path.jpg" % name_video)
path_fig = plot_path(scenelet.skeleton)
legend = ["smooth %g" % args.smooth]
# hack debug
# path_skel2 = os.path.join(args.dir, os.pardir, 'skel_lobby7_nosmooth.json')
# if os.path.exists(path_skel2):
# skel2 = Skeleton.load(path_skel2)
# path_fig = plot_path(skel2, path_fig)
# legend.append('no smooth')
if show:
plt.legend(legend)
path_fig.savefig(out_path_path)
# backup args
path_args = os.path.join(args.dir, os.pardir, 'args_denis.txt')
with open(path_args, 'a') as f_args:
f_args.write("%s %s\n" %
(os.path.basename(sys.executable), " ".join(argv)))
# save 2D detections to file
if args.postfix == 'unannot':
path_skel_ours_2d = os.path.join(
args.dir, os.pardir, "skel_%s_2d_%02d.json" % (name_video, 0))
sclt_2d = Scenelet(skel_ours_2d_all)
print('Saving {} to {}'.format(len(skel_ours_2d_all.get_frames()),
path_skel_ours_2d))
sclt_2d.skeleton.aux_info = {}
sclt_2d.save(path_skel_ours_2d)
else:
print(args.postfix)
logging.info("Saving images...")
if len(images) and len(out_images):
path_out_images = os.path.join(args.dir, os.pardir, 'color')
try:
os.makedirs(path_out_images)
except OSError:
pass
visible_f_max_log = np.log(visible_f_max)
frames = list(out_images.keys())
for frame_id in range(frames[0], frames[-1] + 1):
im = out_images[frame_id] if frame_id in out_images \
else cv2.cvtColor(images[frame_id], cv2.COLOR_BGR2RGB)
for actor_id in range(args.n_actors):
if frame_id in visible_f[actor_id]:
frame_id2 = skel_ours_2d_all.unmod_frame_id(
frame_id=frame_id,
actor_id=actor_id,
frames_mod=skel_ours_2d_all.frames_mod)
for joint, is_vis in visible_f[actor_id][frame_id].items():
p2d = skel_ours_2d_all.get_joint_3d(joint,
frame_id=frame_id2)
# radius = np.log(is_vis) / visible_f_max_log
# lg.debug("r0: %g" % radius)
# radius = np.exp(np.log(is_vis) / visible_f_max_log)
# lg.debug("radius is %g" % radius)
vis_bool = True
if skel_ours_2d_all.has_visible(frame_id=frame_id2,
joint_id=joint):
vis_bool &= skel_ours_2d_all.is_visible(
frame_id2, joint)
radius = abs(np.log(is_vis / 0.1 + 1e-6))
if not np.isnan(radius):
p2d = (int(round(p2d[0])), int(round(p2d[1])))
cv2.circle(im,
center=p2d,
radius=int(round(radius)),
color=(1., 1., 1., 0.5),
thickness=1)
conf = get_conf_thresholded(conf=is_vis,
thresh_log_conf=None,
dtype_np=np.float32)
if conf > 0.5:
cv2.putText(img=im,
text=Joint(joint).get_name(),
org=p2d,
fontFace=1,
fontScale=1,
color=(10., 150., 10., 100.))
# lg.debug("set visibility to %g, radius %g" % (is_vis, radius))
# if frame_id in out_images:
scale = (shape_orig[1] / float(im.shape[1]),
shape_orig[0] / float(im.shape[0]))
cv2.imwrite(
os.path.join(path_out_images, "color_%05d.jpg" % frame_id),
cv2.resize(im, (0, 0),
fx=scale[0],
fy=scale[1],
interpolation=cv2.INTER_CUBIC))
# else:
# fname = "color_%05d.jpg" % frame_id
# shutil.copyfile(
# os.path.join(path_images, fname),
# os.path.join(path_out_images, fname))
lg.info("Wrote images to %s/" % path_out_images)
def evaluate(named_solution,
sclt_gt,
sclt_gt_2d,
frame_ids,
path_scene,
stats,
actions=None,
scale=100.):
"""
:param named_solution:
:param sclt_gt:
:param sclt_gt_2d:
:param frame_ids:
:param path_scene:
:param stats:
:param actions:
:param scale: scale from meter to cm
:return:
"""
p_intrinsics = os.path.join(path_scene, 'intrinsics.json')
intrinsics = np.array(json.load(open(p_intrinsics, 'r')), dtype=np.float32)
print('Loading {}'.format(named_solution.path))
sclt_sol = Scenelet.load(named_solution.path)
sclt_sol.skeleton._visibility.clear()
sclt_sol.skeleton._confidence.clear()
sclt_sol.skeleton._forwards.clear()
sclt_sol.skeleton = Skeleton.resample(sclt_sol.skeleton)
err_3d = Comparison(title='World',
path_scene=path_scene,
name_method=named_solution.name_method,
stats=stats)
err_3d_local = Comparison(title='Local',
path_scene=path_scene,
name_method=named_solution.name_method,
stats=stats)
err_2d = Comparison(title='2D',
path_scene=path_scene,
name_method=named_solution.name_method,
stats=stats)
occlusion = sclt_gt.aux_info['occluded']
missing = {'method': [], 'gt': []}
for frame_id in frame_ids:
try:
entry = sclt_sol.skeleton.get_pose(frame_id=frame_id)
except KeyError:
missing['method'].append(frame_id)
continue
if actions is not None and frame_id in actions \
and actions[frame_id] == 'walking':
print('Skipping non-interactive frame {} {}'.format(
frame_id, actions[frame_id]))
continue
# 3D
gt = sclt_gt.skeleton.get_pose(frame_id=frame_id)
occluded = occlusion['{:d}'.format(frame_id)]
err_3d.add(entry=entry,
gt=gt,
frame_id=frame_id,
scale=scale,
occluded=occluded)
# Local 3D
local_entry = entry - entry[:, Joint.PELV:Joint.PELV + 1]
local_gt = gt - gt[:, Joint.PELV:Joint.PELV + 1]
err_3d_local.add(entry=local_entry,
gt=local_gt,
frame_id=frame_id,
scale=scale,
occluded=occluded)
#
# GT 2D
#
gt_2d = sclt_gt_2d.skeleton.get_pose(frame_id=frame_id)
entry_2d = entry[:2, :] / entry[2, :]
entry_2d[0, :] *= intrinsics[0, 0]
entry_2d[1, :] *= intrinsics[1, 1]
entry_2d[0, :] += intrinsics[0, 2]
entry_2d[1, :] += intrinsics[1, 2]
err_2d.add(entry=entry_2d,
gt=gt_2d[:2, :],
frame_id=frame_id,
occluded=occluded)
# stats.paint(path_dest=os.path.join(path_scene, 'debug_eval'))
mn, mx = np.min(sclt_gt.skeleton.poses, axis=(0, 2)), \
np.max(sclt_gt.skeleton.poses, axis=(0, 2))
err_3d.dimensions = (mx - mn) * scale
assert len(missing['method']) < len(frame_ids) / 2, (missing, frame_ids)
return OrderedDict({
err_3d.title: err_3d,
err_3d_local.title: err_3d_local,
err_2d.title: err_2d,
'_missing': missing
})
def show_folder(argv):
# python3 stealth/pose/fit_full_video.py --show /home/amonszpa/workspace/stealth/data/video_recordings/scenelets/lobby15 opt1
# python3 stealth/pose/visualization/show_charness_scores.py --show /media/data/amonszpa/stealth/shared/video_recordings/library1 -o opt1
pjoin = os.path.join
parser = argparse.ArgumentParser("Fit full video")
parser.add_argument('--show', action='store_true')
parser.add_argument("video", type=argparse_check_exists, help="Input path")
parser.add_argument(
'-o',
'--opt-folder',
help="Which optimization output to process. Default: opt1",
default='opt1')
parser.add_argument("--window-size",
type=int,
help="Window size in frames.",
default=20)
args = parser.parse_args(argv)
d = os.path.join(args.video, args.opt_folder)
assert os.path.exists(d), "does not exist: %s" % d
# parse video path
if args.video.endswith(os.sep):
args.video = args.video[:-1]
name_query = os.path.split(args.video)[-1]
print("split: %s" % repr(os.path.split(args.video)))
p_query = pjoin(args.video, "skel_%s_unannot.json" % name_query) \
if os.path.isdir(args.video) else args.video
assert p_query.endswith('.json'), "Need a skeleton file"
# load initial video path (local poses)
query = Scenelet.load(p_query, no_obj=True)
frame_ids = query.skeleton.get_frames()
centroids = Skeleton.get_resampled_centroids(start=frame_ids[0],
end=frame_ids[-1],
old_frame_ids=frame_ids,
poses=query.skeleton.poses)
depths_times_charnesses = []
skeleton = Skeleton()
depths = []
skeleton.charness_poses = {} # this is in Scenelet incorrectly...
skeleton.score_fit = {} # inventing this now
skeleton.score_reproj = {} # inventing this now
for p in sorted(os.listdir(d)):
d_time = pjoin(d, p)
if not os.path.isdir(d_time):
continue
p_skel = next(
f for f in os.listdir(d_time) if os.path.isfile(pjoin(d_time, f))
and f.startswith('skel') and f.endswith('json') and '_00' in f)
sclt = Scenelet.load(pjoin(d_time, p_skel))
mn, mx = sclt.skeleton.get_frames_min_max()
frame_id = mn + (mx - mn) // 2
if query.skeleton.has_pose(frame_id):
pos_3d = query.skeleton.get_centroid_3d(frame_id)
else:
lin_id = frame_id - frame_ids[0]
pos_3d = centroids[lin_id, :]
# put centroid for each joint
skeleton.set_pose(frame_id=frame_id,
pose=np.tile(pos_3d[:, None], (1, 16)))
with open(pjoin(d_time, 'avg_charness.json')) as fch:
data = json.load(fch)
set_or_max(skeleton.charness_poses, frame_id, data['avg_charness'])
# if frame_id in skeleton.charness_poses:
# lg.warning("Maxing charness at frame %d" % frame_id)
# skeleton.charness_poses[frame_id] = max(
# skeleton.charness_poses[frame_id], data['avg_charness'])
# else:
# skeleton.charness_poses[frame_id] = data['avg_charness']
# fit scores
if 'score_fit' in sclt.aux_info:
set_or_max(skeleton.score_fit, frame_id,
sclt.aux_info['score_fit'])
else:
set_or_max(skeleton.score_fit, frame_id, 0.)
if 'score_reproj' in sclt.aux_info:
set_or_max(skeleton.score_reproj, frame_id,
sclt.aux_info['score_reproj'])
else:
set_or_max(skeleton.score_reproj, frame_id, 0.)
fig = plt.figure(figsize=(16, 12), dpi=100)
ax = fig.add_subplot(121, aspect='equal')
X = [] # skeleton x
Z = [] # skeleton z (depth)
C = [] # charness
F = [] # score_fit
R = [] # score_reproj
T = [] # times
for frame_id in skeleton.get_frames():
c = skeleton.get_joint_3d(6, frame_id=frame_id)
X.append(c[0])
Z.append(c[2])
C.append(skeleton.charness_poses[frame_id])
F.append(skeleton.score_fit[frame_id])
R.append(skeleton.score_reproj[frame_id])
T.append(frame_id)
ax.plot(X, Z, 'k--')
for frame_id in skeleton.get_frames():
if frame_id % 5:
continue
c = skeleton.get_joint_3d(6, frame_id=frame_id)
ax.annotate("%d" % frame_id, xy=(c[0], c[2]), zorder=5)
cax = ax.scatter(X, Z, c=C, cmap='jet', zorder=5)
fig.colorbar(cax)
z_lim = (min(Z), max(Z))
z_span = (z_lim[1] - z_lim[0]) // 2
x_lim = min(X), max(X)
x_span = (x_lim[1] - x_lim[0]) // 2
pad = .5
dspan = z_span - x_span
if dspan > 0:
ax.set_xlim(x_lim[0] - dspan - pad, x_lim[1] + dspan + pad)
ax.set_ylim(z_lim[0] - pad, z_lim[1] + pad)
else:
ax.set_xlim(x_lim[0] - pad, x_lim[1] + pad)
ax.set_ylim(z_lim[0] + dspan - pad, z_lim[1] - dspan + pad)
ax.set_title('Fit score weighted characteristicness\ndisplayed at '
'interpolated initial path position')
ax = fig.add_subplot(122)
ax.plot(T, C, 'x--', label='max charness')
charness_threshes = [0.4, 0.35, 0.3]
mn_thr_charness = min(charness_threshes)
mx_thr_charness = max(charness_threshes)
for ct in charness_threshes:
ax.plot([T[0], T[-1]], [ct, ct], 'r')
ax.annotate("charness %g" % ct, xy=(T[0], ct + 0.005))
charness_sorted = sorted([(fid, c)
for fid, c in skeleton.charness_poses.items()],
key=lambda e: e[1])
to_show = []
# Fitness
divisor = 5.
F_ = -np.log10(F) / divisor
print(F_)
ax.plot(T, F_, 'x--', label="-log_10(score) / %.0f" % divisor)
mx_F_ = np.percentile(F_, 90) # np.max(F_)
for i, (t, f) in enumerate(zip(T, F_)):
if f > mx_F_ or any(C[i] > ct for ct in charness_threshes):
to_show.append(i)
# ax.annotate("%.4f" % (F[i]), xy=(t, f), xytext=(t+4, f-0.02),
# arrowprops=dict(facecolor='none', shrink=0.03))
# charness
# ax.annotate("%.3f\n#%d" % (C[i], t), xy=(t, C[i]),
# xytext=(t-10, C[i]-0.02),
# arrowprops=dict(facecolor='none', shrink=0.03))
windows = [] # [(t_start, t_max, t_end), ...]
crossings = {}
# Reproj
R_ = -np.log10(R) / divisor
# ax.plot(T, R_, 'x--', label="-log_10(score reproj) / %.0f" % divisor)
mx_R_ = np.max(R_)
is_above = [False for _ in charness_threshes]
mx_above = []
for i, (t, r) in enumerate(zip(T, R_)):
# if i in to_show:
# ax.annotate("%.4f" % (R[i]), xy=(t, r), xytext=(t-10, r+0.02),
# arrowprops=dict(facecolor='none', shrink=0.03))
# ax.annotate("%d" % t, xy=(t, r - 0.01))
if (i + 1 < len(C)) and (C[i] > C[i + 1]) and (C[i] > mn_thr_charness):
mx_above.append((C[i], t))
for thr_i, thr in enumerate(charness_threshes):
if (C[i] > thr) != is_above[thr_i] \
or (C[i] > mx_thr_charness and not is_above[thr_i]):
step = 15 * (len(charness_threshes) - thr_i) \
if is_above[thr_i] \
else -15 * thr_i
if is_above[thr_i]:
if 'down' not in crossings:
crossings['down'] = (C[i], t)
# else:
# assert crossings['down'][0] > C[i], (crossings['down'][0], C[i])
else:
if 'up' not in crossings:
crossings['up'] = (C[i - 1], t)
elif crossings['up'][0] < C[i - 1]:
crossings['up'] = (C[i - 1], t)
# ax.annotate("%.3f\n#%d" % (C[i], t), xy=(t, C[i]),
# xytext=(t + step, C[i]-0.1),
# arrowprops=dict(facecolor='none', shrink=0.03))
if C[i] < mn_thr_charness and is_above[thr_i]:
try:
c, t = max((e for e in mx_above), key=lambda e: e[0])
ax.annotate("%.3f\n#%d" % (c, t),
xy=(t, c),
xytext=(t + step, c + 0.1),
arrowprops=dict(facecolor='none',
shrink=0.03))
mx_above = []
windows.append(
(crossings['up'][1], t, crossings['down'][1]))
except (KeyError, ValueError):
lg.warning("Can't find gap: %s, %s" %
(crossings, mx_above))
crossings = {}
is_above[thr_i] = C[i] > thr
break
for crossing in windows:
for i, t in enumerate(crossing):
c = skeleton.charness_poses[t]
step = -15 + i * 10
ax.annotate("%.3f\n#%d" % (c, t),
xy=(t, c),
xytext=(t + step, c - 0.1),
arrowprops=dict(facecolor='none', shrink=0.03))
# extract_gaps([args.video])
# labels
ax.set_title("Scores and charness w.r.t time: max charness: #%d %g" %
(charness_sorted[-1][0], charness_sorted[-1][1]))
ax.set_xlabel('integer time')
ax.legend(loc='lower right')
ax.grid(True)
ax.yaxis.grid(which='both')
ax.xaxis.set_ticks(np.arange(T[0] - 1, T[-1] + 1, 5))
ax.set_yticks([])
ax.set_ylim(0., 1.)
ax.set_ylabel('higher is better')
plt.suptitle("%s" % name_query)
with open(os.path.join(d, 'charness_rank.csv'), 'w') as fout:
fout.write("frame_id,charness\n")
for fid_charness in reversed(charness_sorted):
fout.write("{:d},{:g}\n".format(*fid_charness))
print(fid_charness)
# plt.show()
p_out = os.path.join(d, 'charnesses.svg')
plt.savefig(p_out)
lg.debug("saved to %s" % p_out)
def to_skeleton(self, pose_ids, skeleton3d):
"""Use 'pose_ids' to convert back to an ordered Skeleton.
Args:
pose_ids (Dict[int, Dict[int, int]]):
{frame_id => {actor_id => pose_id}}.
skeleton3d (Skeleton):
Skeleton containing 3D poses.
Returns:
out2d (Skeleton):
2D skeleton with sorted actors.
out3d (Skeleton):
3D skeleton with sorted actors.
"""
skeleton = self._skeleton
n_actors = skeleton.n_actors
frames_mod = skeleton.frames_mod
min_frame_id = skeleton.min_frame_id
out2d = copy.deepcopy(skeleton)
out2d.clear_poses()
out3d = copy.deepcopy(skeleton3d)
out3d.clear_poses()
for frame_id in pose_ids:
for actor_id in range(n_actors):
# expanded frame_id
frame_id2_dst = Skeleton.unmod_frame_id(frame_id=frame_id,
actor_id=actor_id,
frames_mod=frames_mod)
#
# checks
#
assert (actor_id != 0) ^ (frame_id2_dst == frame_id), "no"
frame_id_mod = out3d.mod_frame_id(frame_id=frame_id2_dst)
assert frame_id_mod == frame_id, \
"No: %d %d %d" % (frame_id, frame_id2_dst, frame_id_mod)
actor_id2 = out3d.get_actor_id(frame_id2_dst)
assert actor_id2 == actor_id, \
"No: %s %s" % (actor_id, actor_id2)
#
# Work
#
# which pose explains this actor in this frame
pose_id = pose_ids[frame_id][actor_id]
# check, if actor found
if pose_id < 0:
continue
actor_and_frame_ids = self._pose_ids[frame_id][pose_id]
assert actor_and_frame_ids.frame_id == frame_id
# assert actor_and_frame_ids.actor_id == actor_id
frame_id2_src = actor_and_frame_ids.frame_id2
# 3D pose
pose3d = skeleton3d.get_pose(frame_id=frame_id2_src) # type: np.ndarray
time = skeleton3d.get_time(frame_id=frame_id2_src)
out3d.set_pose(frame_id=frame_id2_dst, pose=pose3d, time=time)
# 2D pose
pose2d = skeleton.get_pose(frame_id=frame_id2_src)
assert skeleton.get_time(frame_id=frame_id2_src) == time, \
"Time mismatch: %g %g" \
% (skeleton.get_time(frame_id=frame_id2_src), time)
out2d.set_pose(frame_id=frame_id2_dst, pose=pose2d, time=time)
# confidence
for jid, conf in skeleton.confidence[frame_id2_src].items():
out3d.set_confidence(frame_id=frame_id2_dst,
joint=jid, confidence=conf)
out3d.set_visible(frame_id=frame_id2_dst, joint=jid,
visible=conf > 0.5)
out2d.set_confidence(frame_id=frame_id2_dst,
joint=jid, confidence=conf)
out2d.set_visible(frame_id=frame_id2_dst, joint=jid,
visible=conf > 0.5)
# Testing
assert out2d.is_confidence_normalized()
assert out3d.is_confidence_normalized()
return out2d, out3d