def select_gaps(argv):
pjoin = os.path.join
parser = argparse.ArgumentParser("Select gaps")
parser = parser_video(parser)
parser.add_argument('--select-gaps', action='store_true')
parser.add_argument("-s",
"--d-scenelets",
dest='s',
type=argparse_check_exists,
help="Folder containing original PiGraphs scenelets")
args = parser.parse_args(argv)
d = os.path.join(args.video, args.d)
assert os.path.exists(d), "does not exist: %s" % d
if os.path.isdir(args.video):
if args.video.endswith(os.sep):
args.video = args.video[:-1]
name_query = args.video.split(os.sep)[-1]
assert len(name_query), args.video.split(os.sep)
p_query = pjoin(args.video, "skel_%s_unannot.json" % name_query) \
if os.path.isdir(args.video) else args.video
assert p_query.endswith('.json'), "Need a skeleton file"
# load initial video path (local poses)
query = Scenelet.load(p_query, no_obj=True)
frame_ids = query.skeleton.get_frames()
centroids = Skeleton.get_resampled_centroids(start=frame_ids[0],
end=frame_ids[-1],
old_frame_ids=frame_ids,
poses=query.skeleton.poses)
depths_times_charnesses = []
skeleton = Skeleton()
depths = []
for p in sorted(os.listdir(d)):
ch = float(p.split('charness')[1][1:])
d_time = pjoin(d, p)
p_skel = next(
f for f in os.listdir(d_time) if os.path.isfile(pjoin(d_time, f))
and f.startswith('skel') and f.endswith('json') and '_00' in f)
sclt = Scenelet.load(pjoin(d_time, p_skel))
mn, mx = sclt.skeleton.get_frames_min_max()
frame_id = mn + (mx - mn) // 2
if query.skeleton.has_pose(frame_id):
pos_3d = query.skeleton.get_centroid_3d(frame_id)
else:
lin_id = frame_id - frame_ids[0]
pos_3d = centroids[lin_id, :]
depth = np.linalg.norm(pos_3d)
depths.append(depth)
depths_times_charnesses.append(
DepthTimeCharness(depth=depth, frame_id=frame_id, charness=ch))
hist, bin_edges = np.histogram(depths, bins=5)
lg.debug("hist: %s" % hist)
lg.debug("edges: %s" % bin_edges)
def get_pose_distance(query3d, path_match, gap):
skel_q = query3d.skeleton
mid_frame = gap[0] + (gap[1] - gap[0]) // 2
frame_q = min((frame_id for frame_id in skel_q.get_frames()),
key=lambda frame_id: abs(frame_id - mid_frame))
time = skel_q.get_time(frame_id=frame_q)
print('Closest to {} is {} with time {}'.format(mid_frame, frame_q, time))
match = Scenelet.load(path_match)
skel_m = match.skeleton
frame_m = skel_m.find_time(time=time)
time_m = skel_m.get_time(frame_id=frame_m)
print('Closest match is {} with time {}'.format(frame_m, time_m))
diffs = []
for frame_id in range(frame_q - 1, frame_q + 2):
if not skel_q.has_pose(frame_id):
print('Skipping frame_id {} in query because missing'.format(
frame_id))
continue
_time_q = skel_q.get_time(frame_id=frame_id)
_frame_m = skel_m.find_time(time=_time_q)
_time_m = skel_m.get_time(frame_id=_frame_m)
if abs(_time_m - _time_q) > 1.:
print('Skipping matched time of {} because too far from {}'.format(
_time_m, _time_q))
continue
pose_q = skel_q.get_pose(frame_id=frame_id)
pose_m = skel_m.get_pose(frame_id=_frame_m)
diff = get_pose_diff(pose_q=pose_q, pose_m=pose_m)
print('Diff: {}'.format(diff))
diffs.append(diff)
# return np.mean(diffs), len(diffs) > 0
return np.max(diffs), len(diffs) > 0
def ipol_ik(p_scenelet, postfix):
"""Uses Blender's IK engine to interpolate 3D joint positions in time.
Sphere positions are tied to Rig endpoints, and these endpoints
will be as close as possible to the noisy targets. Hence the Sphere
positions are the ones we save as interpolated 3D joint positions.
The method will try to preserve all, non-skeleton related
information in the scenelet by only overwriting its skeleton.
Visibility and confidence will be discarded (TODO:?).
Args:
p_scenelet (str):
Path to scenelet containing skeleton to interpolate.
postfix (str):
Tag to append to skeleton name before the json extension.
Returns:
o_path (str): Path, where it saved the interpolated version.
"""
# Load skeleton
assert os.path.exists(p_scenelet), "Does not exist: %s" % p_scenelet
scenelet = Scenelet.load(p_scenelet)
# Prepare animation time
bpy.context.scene.render.fps = 10
bpy.context.scene.render.fps_base = 1
# Call rig importer
import_skeleton_animation(
skeleton=scenelet.skeleton,
name='Output',
add_camera=False,
add_trajectory=False,
time_scale=0.1,
skeleton_transparency=0,
)
# Move head back to top (2D marker is at tip of head, but visually,
# it looks better, if it's rendered with the center of the head around
# the nose)
for obj in bpy.data.objects:
if obj.name.endswith("HEAD.Sphere"):
cnstr = next(c for c in obj.constraints
if c.type == 'COPY_LOCATION')
cnstr.head_tail = 1.
# Extract Sphere endpoints as the new skeleton positions
scenelet.skeleton = extract_skeleton(scene=bpy.context.scene)
# Save to disk
stem, ext = os.path.splitext(p_scenelet)
o_path = "%s_%s.json" % (stem, postfix)
scenelet.save(o_path, save_obj=True)
# Return path
return o_path
def compare_other_to_gt():
scene = bpy.context.scene
assert 'postfix' in globals(), "Need postfix parameter"
name_scene = bpy.path.abspath('//').split(os.sep)[-3]
if 'object_correspondences' not in globals():
object_correspondences = get_correspondences(name_scene, postfix)
p_gt = bpy.path.abspath("//../quant/skel_gt.json")
sclt_gt = Scenelet.load(p_gt)
p_other = bpy.path.abspath("//../quant/skel_%s.json" % postfix)
if True or not os.path.exists(p_other):
frame_ids = find_keyframes(lambda ob: ob.name.startswith('Output.'))
sclt_other = save_annotated_scenelet(sc_path=p_other,
scene=scene,
frame_ids=frame_ids)
else:
sclt_other = Scenelet.load(p_other)
compare_scenes(sclt_other, sclt_gt, object_correspondences)
def read_scenelets(d_scenelets, limit=0):
pjoin = os.path.join
# get full pigraph scenes
p_scenelets = [
pjoin(d_scenelets, d) for d in os.listdir(d_scenelets)
if os.path.isdir(pjoin(d_scenelets, d))
]
# for
p_scenelets = [
pjoin(d, f) for d in p_scenelets for f in os.listdir(d)
if f.startswith('skel') and f.endswith('.json')
]
out = []
for p_scenelet in p_scenelets:
lg.info("Reading %s" % p_scenelet)
out.append(Scenelet.load(p_scenelet))
if limit != 0 and len(out) >= limit:
break
return out
def main(argv):
pjoin = os.path.join
parser = argparse.ArgumentParser("")
parser.add_argument('d', type=_check_exists, help="Input directory")
parser.add_argument('db',
type=_check_exists,
help="Path to scenelets database")
parser.add_argument(
'-o',
'--opt-folder',
help="Which optimization output to process. Default: opt1",
default='opt1')
parser.add_argument('-limit',
type=int,
help="How many scenelets to aggregate.",
default=3)
parser.add_argument('--c-threshold',
help='Distance threshold. Default: 0.3',
type=float,
default=0.3)
parser.add_argument('--d-threshold',
help='Distance threshold. Default: 0.4',
type=float,
default=0.4)
args = parser.parse_args(argv)
path_scenes_root = os.path.normpath(pjoin(args.d, os.pardir))
print("Working with %s" % args.d)
d = pjoin(args.d, args.opt_folder)
name_query = d.split(os.sep)[-2]
query3d = Scenelet.load(
os.path.join(args.d, 'skel_{:s}_unannot.json'.format(name_query)))
n_actors = query3d.skeleton.n_actors # type: int
log = LoggedSequence()
elements = [[], [], []]
spans = []
above = []
for p in sorted(os.listdir(d)):
d_time = pjoin(d, p)
if not os.path.isdir(d_time) or 'bak' in p:
continue
parts = p.split('_')
start = int(parts[0])
end = int(parts[-1])
sum_charness = 0.
sum_weight = 0.
sum_charness_unw = 0.
sum_weight_unw = 0
diff_succ, diff = False, 100.
for f in glob.iglob("%s/skel_%s_*.json" % (d_time, name_query)):
rank = int(os.path.splitext(f)[0].rpartition('_')[-1])
if rank >= args.limit:
continue
# print(f)
data = json.load(open(f, 'r'))
charness = data['charness']
weight = max(0., .1 - data['score_fit'])
sum_charness += weight * charness
sum_weight += weight
sum_charness_unw += charness
elements[sum_weight_unw].append(charness)
sum_weight_unw += 1
if rank == 0:
diff, diff_succ = get_pose_distance(query3d=query3d,
path_match=f,
gap=(start, end))
if sum_weight > 0.:
sum_charness /= sum_weight
if sum_weight_unw > 0.:
sum_charness_unw /= sum_weight_unw
frame_id = (start + end) // 2
actor_id = query3d.skeleton.get_actor_id(frame_id=frame_id) \
if n_actors > 1 else 0
spans.append(
Span(start, end, sum_charness, sum_charness_unw,
actor_id=actor_id))
# check for pose replacement
if diff_succ and diff > args.d_threshold:
# time = query3d.skeleton.get_time(frame_id=frame_id)
above.append(Span2(start=frame_id, end=frame_id, value=diff))
if diff_succ:
log.add_point(value=diff, time=frame_id)
for actor_id in range(n_actors):
cs, span_ids = zip(*[(span.charness, span_id)
for span_id, span in enumerate(spans)
if span.actor_id == actor_id])
cs2 = gaussian_filter(cs, sigma=2.5).tolist()
for smoothed, span_id in zip(cs2, span_ids):
spans[span_id].smoothed_charness = smoothed
plt.figure()
plt.plot(cs, label="orig")
plt.plot([span.smoothed_charness for span in spans], label="smoothed")
# c0 = [span.charness_unw for span in spans]
# plt.plot(c0, 'o-', label="unweighted")
# for sigma in [2.5]:
# cs2 = gaussian_filter(cs, sigma=sigma)
# plt.plot(cs2, '+-', label="%f" % sigma)
# for i, elem in enumerate(elements):
# plt.plot(elem, 'x-', label="cand%d" % i)
plt.legend()
# plt.show()
p_graph = os.path.join(args.d, 'charness2.svg')
plt.savefig(p_graph)
log.plot(path_root=args.d)
spans = sorted(spans, key=lambda s: s.smoothed_charness, reverse=True)
chosen = [spans[0]]
for span_id in range(1, len(spans)):
span = spans[span_id]
overlap = next((c for c in chosen if span.overlaps(c)), None)
if overlap is not None:
# print("Skipping %s because overlaps %s" % (span, overlap))
continue
elif span.start == query3d.skeleton.get_frames()[0] \
or span.end == query3d.skeleton.get_frames()[-1]:
print("Skipping %s because first/last frame." % span)
continue
else:
chosen.append(span)
# Spans to replace, because of quality, if not replaced already
spans_r = group_spans(above=above)
chosen, spans_r_filtered = filter_spans_to_replace(
spans=spans_r, chosen=chosen, c_threshold=args.c_threshold)
print('Filtered: {}\nchosen: {}'.format(spans_r_filtered, chosen))
# keep at least one characteristic pose for each actor
# actors_output = set()
p_cmd = os.path.join(args.d, 'gap_command_new.sh')
with open(p_cmd, 'w') as f:
f.write('BATCH_SIZE=8\n\n')
for actor_id in range(n_actors):
f.write('######\n# Actor {:d}\n######\n\n'.format(actor_id))
for span in chain(chosen, spans_r_filtered):
if not span.actor_id == actor_id:
continue
surr = find_surrounding(span, spans)
cmd = construct_command(name_query=name_query,
span=span,
tc=args.c_threshold,
surr=surr,
path_scenes_root=path_scenes_root,
path_scenelets_db=args.db)
if isinstance(span.charness, float):
f.write("# charness: %g\n" % span.charness)
# Comment out if too low
# if span.charness < args.c_threshold \
# and actor_id in actors_output:
# actors_output.add(actor_id)
f.write('{}\n\n'.format(cmd))
f.write('\n\n')
if __name__ == 'add_constraint_to_spheres':
add_constraint_to_spheres()
elif __name__ == 'save_annotated_scenelet':
assert 'postfix' in globals(), "Need postfix parameter"
print("abspath: %s" % bpy.path.abspath('//'))
name_scene = bpy.path.abspath('//').split(os.sep)[-3]
scene = bpy.context.scene
# Aron commented out in Jan2019
if 'object_correspondences' not in globals():
object_correspondences = get_correspondences(name_scene)
p_file = bpy.path.abspath('//../output/skel_output.json')
assert os.path.exists(p_file), "Does not exist: %s" % p_file
sclt_ours = Scenelet.load(p_file)
p_out = bpy.path.abspath('//../quant/skel_output.json')
sclt_ours.save(p_out, save_obj=True)
p_gt = bpy.path.abspath("//../quant/skel_%s.json" % postfix)
if not os.path.exists(p_gt):
if 'frame_ids' not in globals():
frame_ids = get_frame_ids(name_scene)
# frame_ids = find_keyframes(lambda ob: ob.name.startswith('Output.'))
sclt_gt = save_annotated_scenelet(sc_path=p_gt,
scene=scene,
frame_ids=frame_ids,
frame_multiplier=frame_multiplier,
time_multiplier=time_multiplier)
else:
print("NOT OVERWRITING GT")
def optimize_path(skel_ours,
skel_ours_2d,
images,
intrinsics,
path_skel,
ground_rot,
shape_orig=None,
use_huber=False,
weight_smooth=0.01,
show=False,
frames_ignore=None,
resample=True,
depth_init=10.,
p_constraints=None,
smooth_mode=SmoothMode.ACCEL):
"""Optimize 3D path so that it matches the 2D corresponding observations.
Args:
skel_ours (Skeleton):
3D skeleton from LFD.
skel_ours_2d (Skeleton):
2D feature points from LFD.
images (dict):
Color images for debug, keyed by frame_ids.
camera_name (str):
Initialize intrinsics matrix based on name of camera.
path_skel (str):
Path of input file from LFD on disk, used to create paths for
intermediate result.
shape_orig (tuple):
Height and width of original images before LFD scaled them.
use_huber (bool):
Deprecated.
weight_smooth (float):
Smoothness term weight.
winsorize_limit (float):
Outlier detection parameter.
show (bool):
Show debug visualizations.
frames_ignore (set):
Deprecated.
resample (bool):
Fill in missing poses by interpolating using Blender's IK.
depth_init (float):
Initial depth for LFD poses.
p_constraints (str):
Path to 3D constraints scenelet file.
smooth_mode (SmoothMode):
Smooth velocity or acceleration.
"""
# scale 2D detections to canonical camera coordinates
np_poses_2d = \
skel_ours_2d.poses[:, :2, :] \
- np.expand_dims(intrinsics[:2, 2], axis=1)
np_poses_2d[:, 0, :] /= intrinsics[0, 0]
np_poses_2d[:, 1, :] /= intrinsics[1, 1]
n_frames = skel_ours.poses.shape[0]
np_translation = np.zeros(shape=(n_frames, 3), dtype=np.float32)
np_translation[:, 1] = -1.
np_translation[:, 2] = \
np.random.uniform(-depth_init * 0.25, depth_init * 0.25,
np_translation.shape[0]) \
+ depth_init
np_rotation = np.zeros(shape=(n_frames, 3), dtype=np.float32)
frame_ids = np.array(skel_ours.get_frames(), dtype=np.float32)
np_visibility = skel_ours_2d.get_confidence_matrix(frame_ids=frame_ids,
dtype='f4')
if p_constraints is not None:
sclt_cnstr = Scenelet.load(p_constraints)
np_cnstr_mask = np.zeros(shape=(len(frame_ids),
Joint.get_num_joints()),
dtype=np.float32)
np_cnstr = np.zeros(shape=(len(frame_ids), 3, Joint.get_num_joints()),
dtype=np.float32)
for frame_id, confs in sclt_cnstr.confidence.items():
lin_id = None
for j, conf in confs.items():
if conf > 0.5:
if lin_id is None:
lin_id = next(
lin_id_
for lin_id_, frame_id_ in enumerate(frame_ids)
if frame_id_ == frame_id)
np_cnstr_mask[lin_id, j] = conf
np_cnstr[lin_id, :, j] = \
sclt_cnstr.skeleton.get_joint_3d(
joint_id=j, frame_id=frame_id)
else:
np_cnstr_mask = None
np_cnstr = None
spans = skel_ours.get_actor_empty_frames()
dt = frame_ids[1:].astype(np.float32) \
- frame_ids[:-1].astype(np.float32)
dt_pos_inv = np.reciprocal(dt, dtype=np.float32)
dt_vel_inv = np.divide(np.float32(2.), dt[1:] + dt[:-1])
# ensure smoothness weight multipliers are not affected by
# actor-transitions
if skel_ours.n_actors > 1 and len(spans):
for lin_id in range(len(dt)):
frame_id0 = frame_ids[lin_id]
frame_id1 = frame_ids[lin_id + 1]
span = next((span_ for span_ in spans if span_[0] == frame_id0),
None)
if span is not None:
assert frame_id1 == span[1], "No"
dt[lin_id] = 0.
dt_pos_inv[lin_id] = 0.
dt_vel_inv[lin_id] = 0.
dt_vel_inv[lin_id - 1] = 1. / dt[lin_id - 1]
forwards = np.array([
skel_ours.get_forward(frame_id, estimate_ok=True, k=0)
for frame_id in skel_ours.get_frames()
])
# from alignment import get_angle
# xs = np.hstack((
# np.ones(shape=(len(forwards), 1)),
# np.zeros(shape=(len(forwards), 2))
# ))
# print(xs.shape)
print(forwards.shape)
unit_x = np.array((1., 0., 0.))
np_angles = [-np.arctan2(forward[2], forward[0]) for forward in forwards]
print(forwards, np_angles)
# ank_diff = \
# np.exp(
# -2. * np.max(
# [
# np.linalg.norm(
# (skel_ours.poses[1:, :, joint]
# - skel_ours.poses[:-1, :, joint]).T
# * dt_pos_inv, axis=0
# ).astype(np.float32)
# for joint in {Joint.LANK, Joint.RANK}
# ],
# axis=0
# )
# )
# assert ank_diff.shape == (skel_ours.poses.shape[0]-1,), \
# "Wrong shape: %s" % repr(ank_diff.shape)
# cam_angle = [np.deg2rad(-8.)]
assert np.isclose(ground_rot[1], 0.) and np.isclose(ground_rot[2], 0.), \
"Assumed only x rotation"
# assert ground_rot[0] <= 0, "Negative means looking down, why looknig up?"
cam_angle = [np.deg2rad(ground_rot[0])]
# assert False, "Fixed angle!"
device_name = '/gpu:0' if tf.test.is_gpu_available() else '/cpu:0'
devices = {device_name}
for device in devices:
with Timer(device, verbose=True):
graph = tf.Graph()
with graph.as_default(), tf.device(device):
tf_visibility = tf.Variable(np.tile(np_visibility, (1, 2, 1)),
name='visibility',
trainable=False,
dtype=tf.float32)
tf_dt_pos_inv = \
tf.Variable(np.tile(dt_pos_inv, (1, 3)).reshape(-1, 3),
name='dt_pos_inv', trainable=False,
dtype=tf.float32)
tf_dt_vel_inv = \
tf.constant(np.tile(dt_vel_inv, (1, 3)).reshape(-1, 3),
name='dt_vel_inv', dtype=tf.float32)
# input data
pos_3d_in = tf.Variable(skel_ours.poses.astype(np.float32),
trainable=False,
name='pos_3d_in',
dtype=tf.float32)
pos_2d_in = tf.Variable(np_poses_2d.astype(np.float32),
trainable=False,
name='pos_2d_in',
dtype=tf.float32)
params_camera = tf.Variable(initial_value=cam_angle,
dtype=tf.float32,
trainable=True)
cam_sn = tf.sin(params_camera)
cam_cs = tf.cos(params_camera)
transform_camera = tf.reshape(tf.stack([
1., 0., 0., 0., 0., cam_cs[0], cam_sn[0], 0., 0.,
-cam_sn[0], cam_cs[0], 0., 0., 0., 0., 1.
],
axis=0),
shape=(4, 4))
# 3D translation
translation = tf.Variable(np_translation, name='translation')
# 3D rotation (Euler XYZ)
rotation = tf.Variable(np_rotation, name='rotation')
fw_angles = tf.Variable(np_angles, name='angles')
# rotation around y
my_zeros = tf.zeros((n_frames, 1))
my_ones = tf.ones((n_frames, 1))
c = tf.cos(tf.slice(rotation, [0, 1], [n_frames, 1]))
s = tf.sin(tf.slice(rotation, [0, 1], [n_frames, 1]))
t0 = tf.concat([c, my_zeros, -s, my_zeros], axis=1)
t1 = tf.concat([my_zeros, my_ones, my_zeros, my_zeros], axis=1)
t2 = tf.concat([s, my_zeros, c, my_zeros], axis=1)
t3 = tf.concat([my_zeros, my_zeros, my_zeros, my_ones], axis=1)
transform = tf.stack([t0, t1, t2, t3],
axis=2,
name="transform")
transform = tf.einsum('ij,ajk->aik', transform_camera,
transform)[:, :3, :3]
# transform to 3d
pos_3d = tf.matmul(transform, pos_3d_in) \
+ tf.tile(tf.expand_dims(translation, 2),
[1, 1, int(pos_3d_in.shape[2])])
# constraints
loss_cnstr = None
if np_cnstr is not None:
constraints = tf.Variable(np_cnstr,
trainable=False,
name='constraints',
dtype=tf.float32)
constraints_mask = tf.Variable(np_cnstr_mask,
trainable=False,
name='constraints_mask',
dtype=tf.float32)
cnstr_diff = tf.reduce_sum(tf.squared_difference(
pos_3d, constraints),
axis=1,
name='constraints_difference')
cnstr_diff_masked = tf.multiply(
constraints_mask,
cnstr_diff,
name='constraints_difference_masked')
loss_cnstr = tf.reduce_sum(cnstr_diff_masked,
name='constraints_loss')
# perspective divide
pos_2d = tf.divide(
tf.slice(pos_3d, [0, 0, 0], [n_frames, 2, -1]),
tf.slice(pos_3d, [0, 2, 0], [n_frames, 1, -1]))
if use_huber:
diff = huber_loss(pos_2d_in, pos_2d, 1.)
masked = diff * tf_visibility
loss_reproj = tf.nn.l2_loss(masked)
lg.info("Doing huber on reprojection, NOT translation")
else:
# re-projection loss
diff = pos_2d - pos_2d_in
# mask loss by 2d key-point visibility
masked = diff * tf_visibility
loss_reproj = tf.nn.l2_loss(masked)
lg.info("NOT doing huber")
sys.stderr.write(
"TODO: Move huber to translation, not reconstruction\n")
# translation smoothness
dx = tf.multiply(
x=0.5,
y=tf.add(
pos_3d[1:, :, Joint.LHIP] - pos_3d[:-1, :, Joint.LHIP],
pos_3d[1:, :, Joint.RHIP] - pos_3d[:-1, :, Joint.RHIP],
),
name="average_hip_displacement_3d")
tf_velocity = tf.multiply(dx, tf_dt_pos_inv)
tf_acceleration_z = tf.multiply(x=dx[1:, 2:3] - dx[:-1, 2:3],
y=tf_dt_vel_inv[:, 2:3],
name="acceleration_z")
if smooth_mode == SmoothMode.VELOCITY:
# if GT, use full smoothness to fix 2-frame flicker
if np_cnstr is not None:
print('Smoothing all velocity!')
loss_transl_smooth = \
weight_smooth * tf.nn.l2_loss(tf_velocity)
else: # Normal mode, don't oversmooth screen-space
loss_transl_smooth = \
weight_smooth * tf.nn.l2_loss(tf_velocity[:, 2:3])
elif smooth_mode == SmoothMode.ACCEL:
loss_transl_smooth = \
weight_smooth * tf.nn.l2_loss(tf_acceleration_z)
else:
raise RuntimeError(
'Unknown smooth mode: {}'.format(smooth_mode))
if show:
sqr_accel_z = weight_smooth * tf.square(tf_acceleration_z)
if weight_smooth > 0.:
lg.info("Smoothing in time!")
loss = loss_reproj + loss_transl_smooth
else:
lg.warning("Not smoothing!")
loss = loss_reproj
if loss_cnstr is not None:
loss += 1000 * loss_cnstr
# hip0 = tf.nn.l2_normalize(pos_3d[:-1, :, Joint.RHIP] - pos_3d[:-1, :, Joint.LHIP])
# hip1 = tf.nn.l2_normalize(pos_3d[1:, :, Joint.RHIP] - pos_3d[1:, :, Joint.RHIP])
# dots = tf.reduce_sum(tf.multiply(hip0, hip1), axis=1)
# print(dots)
# loss_dot = tf.nn.l2_loss(1. - dots)
# loss_ang = fw_angles + rotation[:, 1]
# print(loss_ang)
# loss_ang = tf.square(loss_ang[1:] - loss_ang[:-1])
# print(loss_ang)
# two_pi_sqr = tf.constant((2. * 3.14159)**2., dtype=tf.float32)
# print(two_pi_sqr)
# loss_ang = tf.reduce_mean(tf.where(loss_ang > two_pi_sqr, loss_ang - two_pi_sqr, loss_ang))
# print(loss_ang)
# loss += loss_ang
#
# optimize
#
optimizer = ScipyOptimizerInterface(
loss,
var_list=[translation, rotation],
options={'gtol': 1e-12},
var_to_bounds={rotation: (-np.pi / 2., np.pi / 2.)})
with tf.Session(graph=graph) as session:
session.run(tf.global_variables_initializer())
optimizer.minimize(session)
np_pos_3d_out, np_pos_2d_out, np_transl_out, np_masked, \
np_acceleration, np_loss_transl_smooth, np_dt_vel = \
session.run([pos_3d, pos_2d, translation, masked,
tf_acceleration_z, loss_transl_smooth,
tf_dt_vel_inv])
if show:
o_sqr_accel_z = session.run(sqr_accel_z)
o_vel = session.run(tf_velocity)
o_dx = session.run(dx)
o_rot = session.run(rotation)
# o_dx, o_dx2 = session.run([accel_bak, acceleration2])
# assert np.allclose(o_dx, o_dx2), "no"
o_cam = session.run(fetches=[params_camera])
print("camera angle: %s" % np.rad2deg(o_cam[0]))
# o_losses = session.run([loss_reproj, loss_transl_smooth, loss_dot, loss_ang])
o_losses = session.run([loss_reproj, loss_transl_smooth])
print('losses: {}'.format(o_losses))
# o_dots = session.run(dots)
# with open('tmp/dots.txt', 'w') as fout:
# fout.write('\n'.join((str(e) for e in o_dots.tolist())))
fixed_frames = []
# for lin_frame_id in range(np_transl_out.shape[0]):
# if np_transl_out[lin_frame_id, 2] < 0.:
# print("Correcting frame_id %d: %s"
# % (skel_ours.get_lin_id_for_frame_id(lin_frame_id),
# np_transl_out[lin_frame_id, :]))
# if lin_frame_id > 0:
# np_transl_out[lin_frame_id, :] = np_transl_out[lin_frame_id-1, :]
# else:
# np_transl_out[lin_frame_id, :] = np_transl_out[lin_frame_id+1, :]
# fixed_frames.append(lin_frame_id)
# debug_forwards(skel_ours.poses, np_pos_3d_out, o_rot, forwards, np_angles)
# z_jumps = np_pos_3d_out[1:, 2, Joint.PELV] - np_pos_3d_out[:-1, 2, Joint.PELV]
# out = scipy.stats.mstats.winsorize(z_jumps, limits=1.)
# plt.figure()
# plt.plot(pos_3d[:, 2, Joint.PELV])
# plt.show()
# sys.exit(0)
# diff = np.linalg.norm(out - displ, axis=1)
if len(fixed_frames):
print("Re-optimizing...")
with tf.Session(graph=graph) as session:
np_pos_3d_out, np_pos_2d_out, np_transl_out = \
session.run(fetches=[pos_3d, pos_2d, translation],
feed_dict={transform: np_transl_out})
if show:
lim_fr = [105, 115, 135]
fig = plt.figure()
accel_thr = 0. # np.percentile(o_sqr_accel_z, 25)
ax = plt.subplot2grid((2, 2), (0, 0), colspan=2)
# print("np_masked:%s" % np_masked)
# plt.plot(np_masked[:, )
ax.plot(np.linalg.norm(np_acceleration[lim_fr[0]:lim_fr[1]], axis=1),
'--o',
label='accel')
ax.add_artist(Line2D([0, len(o_sqr_accel_z)], [accel_thr, accel_thr]))
# plt.plot(np_dt_vel[:, 0], label='dt velocity')
# plt.plot(np.linalg.norm(np_f_accel, axis=1), '--x', label='f_accel')
# plt.plot(ank_diff, label='ank_diff')
ax.plot(o_sqr_accel_z[lim_fr[0]:lim_fr[1] + 1],
'--x',
label='loss accel_z')
ax.legend()
ax2 = plt.subplot2grid((2, 2), (1, 0), aspect='equal')
ax2.plot(np_pos_3d_out[lim_fr[0]:lim_fr[1] + 1, 0, Joint.PELV],
np_pos_3d_out[lim_fr[0]:lim_fr[1] + 1, 2, Joint.PELV], '--x')
for i, vel in enumerate(o_vel):
if not (lim_fr[0] <= i <= lim_fr[1]):
continue
p0 = np_pos_3d_out[i + 1, [0, 2], Joint.PELV]
p1 = np_pos_3d_out[i, [0, 2], Joint.PELV]
ax2.annotate(
"%f = ((%g - %g) + (%g - %g)) * %g = %g" %
(vel[2], np_pos_3d_out[i + 1, 2, Joint.LHIP],
np_pos_3d_out[i, 2, Joint.LHIP], np_pos_3d_out[i + 1, 2,
Joint.RHIP],
np_pos_3d_out[i, 2, Joint.RHIP], np_dt_vel[i, 2], o_dx[i, 2]),
xy=((p0[0] + p1[0]) / 2., (p0[1] + p1[1]) / 2.))
ax2.set_title('velocities')
ax1 = plt.subplot2grid((2, 2), (1, 1), aspect='equal')
ax1.plot(np_pos_3d_out[lim_fr[0]:lim_fr[1] + 1, 0, Joint.PELV],
np_pos_3d_out[lim_fr[0]:lim_fr[1] + 1, 2, Joint.PELV], '--x')
for i, lacc in enumerate(o_sqr_accel_z):
if not (lim_fr[0] <= i <= lim_fr[1]):
continue
if lacc > accel_thr:
p0 = np_pos_3d_out[i + 1, [0, 2], Joint.PELV]
ax1.annotate("%.3f" % np_acceleration[i], xy=(p0[0], p0[1]))
ax.annotate("%.3f" % np.log10(lacc),
xy=(i - lim_fr[0], abs(np_acceleration[i])))
ax1.set_title('accelerations')
plt.show()
np.set_printoptions(linewidth=200)
np_pos_2d_out[:, 0, :] *= intrinsics[0, 0]
np_pos_2d_out[:, 1, :] *= intrinsics[1, 1]
np_pos_2d_out[:, 0, :] += intrinsics[0, 2]
np_pos_2d_out[:, 1, :] += intrinsics[1, 2]
np_poses_2d[:, 0, :] *= intrinsics[0, 0]
np_poses_2d[:, 1, :] *= intrinsics[1, 1]
np_poses_2d[:, 0, :] += intrinsics[0, 2]
np_poses_2d[:, 1, :] += intrinsics[1, 2]
out_images = {}
if shape_orig is not None:
frames_2d = skel_ours_2d.get_frames()
for frame_id2 in frames_2d:
try:
lin_frame_id = skel_ours_2d.get_lin_id_for_frame_id(frame_id2)
except KeyError:
lin_frame_id = None
frame_id = skel_ours_2d.mod_frame_id(frame_id=frame_id2)
im = None
if frame_id in out_images:
im = out_images[frame_id]
elif len(images):
if frame_id not in images:
lg.warning("Not enough images, the video was probably cut "
"after LiftingFromTheDeep was run.")
continue
im = copy.deepcopy(images[frame_id])
im = cv2.cvtColor(im, cv2.COLOR_RGB2BGR)
else:
im = np.zeros(
(shape_orig[0].astype(int), shape_orig[1].astype(int), 3),
dtype='i1')
if lin_frame_id is not None:
for jid in range(np_pos_2d_out.shape[2]):
if skel_ours_2d.is_visible(frame_id2, jid):
p2d = tuple(np_pos_2d_out[lin_frame_id, :,
jid].astype(int).tolist())
p2d_det = tuple(np_poses_2d[lin_frame_id, :,
jid].astype(int).tolist())
cv2.line(im,
p2d,
p2d_det,
color=(100, 100, 100),
thickness=3)
cv2.circle(im,
p2d,
radius=3,
color=(0, 0, 200),
thickness=-1)
cv2.circle(im,
p2d_det,
radius=3,
color=(0, 200, 0),
thickness=-1)
out_images[frame_id] = im
# cv2.imshow("Out", im)
# cv2.waitKey(50)
if False:
# visualize
fig = plt.figure()
ax = fig.gca(projection='3d')
for frame_id in range(0, np_pos_3d_out.shape[0], 1):
j = Joint.PELV
ax.scatter(np_pos_3d_out[frame_id, 0, j],
np_pos_3d_out[frame_id, 2, j],
-np_pos_3d_out[frame_id, 1, j],
marker='o')
# smallest = np_pos_3d_out.min()
# largest = np_pos_3d_out.max()
ax.set_xlim3d(-5., 5.)
ax.set_xlabel('x')
ax.set_ylim3d(-5., 5.)
ax.set_ylabel('y')
ax.set_zlim3d(-5., 5.)
ax.set_zlabel('z')
if False:
# visualize
fig = plt.figure()
ax = fig.gca(projection='3d')
for frame_id in range(0, np_pos_3d_out.shape[0], 1):
for j in range(np_pos_3d_out.shape[2]):
ax.scatter(np_pos_3d_out[frame_id, 0, j],
np_pos_3d_out[frame_id, 2, j],
-np_pos_3d_out[frame_id, 1, j],
marker='o')
# smallest = np_pos_3d_out.min()
# largest = np_pos_3d_out.max()
ax.set_xlim3d(-5., 5.)
ax.set_xlabel('x')
ax.set_ylim3d(-5., 5.)
ax.set_ylabel('y')
ax.set_zlim3d(-5., 5.)
ax.set_zlabel('z')
plt.show()
assert all(a == b
for a, b in zip(skel_ours.poses.shape, np_pos_3d_out.shape)), \
"no"
skel_ours.poses = np_pos_3d_out
return skel_ours, out_images, intrinsics
def main(argv):
parser = argparse.ArgumentParser(
"Filter initial path based on distance to full fit")
parser.add_argument('skel', help="Skeleton file to filter", type=str)
parser.add_argument('--threshold',
help='Distance threshold. Default: 0.4',
type=float,
default=0.4)
args = parser.parse_args(argv)
lower_body = [
Joint.LKNE, Joint.RKNE, Joint.LANK, Joint.RANK, Joint.LHIP, Joint.RHIP
]
print(args.skel)
p_root = os.path.dirname(args.skel)
p_fit = os.path.join(p_root, 'opt1')
assert os.path.isdir(p_fit), p_fit
query = Scenelet.load(args.skel)
out = Skeleton()
data = []
x = []
y = []
y2 = []
for d_ in sorted(os.listdir(p_fit)):
d = os.path.join(p_fit, d_)
pattern = os.path.join(d, 'skel_*.json')
for f in sorted(glob.iglob(pattern)):
print(f)
assert '00' in f, f
sclt = Scenelet.load(f)
frames = sclt.skeleton.get_frames()
mid_frame = frames[len(frames) // 2]
time = sclt.skeleton.get_time(mid_frame)
q_frame_id = query.skeleton.find_time(time)
q_time = query.skeleton.get_time(q_frame_id)
print(time, q_time, f)
q_pose = query.skeleton.get_pose(q_frame_id)
pose = sclt.skeleton.get_pose(mid_frame)
pose[[0, 2]] -= (pose[:, Joint.PELV:Joint.PELV + 1] -
q_pose[:, Joint.PELV:Joint.PELV + 1])[[0, 2]]
diff = np.mean(
np.linalg.norm(q_pose[:, lower_body] - pose[:, lower_body],
axis=0))
print(q_frame_id, time, diff)
y.append(diff)
x.append(q_frame_id)
data.append((q_frame_id, diff, time))
if query.skeleton.has_pose(q_frame_id - 1):
tmp_pose = copy.deepcopy(q_pose)
tmp_pose -= tmp_pose[:, Joint.PELV:Joint.PELV +
1] - query.skeleton.get_pose(
q_frame_id -
1)[:, Joint.PELV:Joint.PELV + 1]
y2.append(
np.mean(
np.linalg.norm(pose[:, lower_body] -
tmp_pose[:, lower_body],
axis=0)))
else:
y2.append(0.)
out.set_pose(frame_id=q_frame_id, time=q_time, pose=pose)
break
data = smooth(data)
plt.plot(x, y, 'x--', label='Distance to best Kinect fit\'s center frame')
plt.plot(x, y2, 'o--', label='Distance to prev pose')
plt.plot([d[0] for d in data], [d[1] for d in data],
'o--',
label='Smoothed')
plt.xlabel('Time (s)')
plt.ylabel('Sum local squared distance')
plt.legend()
plt.savefig(os.path.join(p_root, 'tmp.pdf'))
Scenelet(skeleton=out).save(os.path.join(p_root, 'skel_tmp.json'))
above = []
prev_frame_id = None
for frame_id, dist, time in data:
# assert prev_frame_id is None or frame_id != prev_frame_id, \
# 'No: {}'.format(frame_id)
if dist > args.threshold:
above.append(
Span2(start=frame_id, end=frame_id, value=dist, time=time))
prev_frame_id = frame_id
spans = [copy.deepcopy(above[0])]
it = iter(above)
next(it)
prev_frame_id = above[0].start
for span2 in it:
frame_id = span2.start
if prev_frame_id + 1 < frame_id:
# span = spans[-1]
# spans[-1] = span[0], prev_frame_id, span[2]
spans[-1].end = prev_frame_id
spans.append(
Span2(start=frame_id,
end=frame_id,
time=None,
value=span2.value))
else:
print(prev_frame_id, frame_id)
prev_frame_id = frame_id
spans[-1].end = prev_frame_id
print("Need replacement: {}".format(above))
print("Need replacement2: {}".format(spans))
def main(argv):
np.set_printoptions(suppress=True, linewidth=200)
pjoin = os.path.join
parser = argparse.ArgumentParser("matcher")
parser.add_argument("d_scenelets",
type=argparse_check_exists,
help="Folder containing scenelets")
parser.add_argument("video", type=argparse_check_exists, help="Input path")
parser.add_argument("--gap-size-limit",
type=int,
help="Smallest gap size to still explain")
args = parser.parse_args(argv)
d_query = os.path.dirname(args.video)
# 2d keypoint rescale
p_im = pjoin(d_query, 'origjpg', 'color_00100.jpg')
im_ = cv2.imread(p_im)
shape_orig = im_.shape
scale_2d = shape_orig[0] / float(INPUT_SIZE)
query = Scenelet.load(args.video, no_obj=True)
tr_ground = np.array(query.aux_info['ground'], dtype=np.float32)
print("tr: %s" % tr_ground)
name_query = os.path.basename(args.video).split('_')[1]
query_2d = Scenelet.load(pjoin(d_query,
"skel_%s_2d_00.json" % name_query)).skeleton
p_intr = pjoin(d_query, 'intrinsics.json')
intr = np.array(json.load(open(p_intr, 'r')), dtype=np.float32)
lg.debug("intr: %s" % intr)
gaps = find_gaps(query.skeleton, min_pad=1)
p_scenelets_pickle = pjoin(args.d_scenelets, 'match_gap_scenelets.pickle')
if os.path.exists(p_scenelets_pickle):
scenelets = pickle_load(open(p_scenelets_pickle, 'rb'))
else:
scenelets = read_scenelets(args.d_scenelets, limit=0)
pickle.dump(scenelets, open(p_scenelets_pickle, 'wb'))
p_out_sclts = pjoin(d_query, 'fill')
if os.path.exists(p_out_sclts):
shutil.rmtree(p_out_sclts)
os.makedirs(p_out_sclts)
times = []
for gap_id, gap in enumerate(gaps):
lg.debug("gap is %s" % repr(gap))
if gap[1] - gap[0] < args.gap_size_limit:
continue
with Timer("gap %d" % gap_id) as timer:
chosen = []
for sc_id, sclt in enumerate(scenelets):
lg.info("scenelet %d / %d" % (sc_id, len(scenelets)))
sclt = scenelets[sc_id]
ground_obj = next(ob for ob in sclt.objects.values()
if ob.label == 'floor')
ground_part = ground_obj.get_part_by_name('floor')
lg.debug("floor: %s" % ground_part)
ground_transform = ground_part.obb.as_transform()
lg.debug("floor: %s" % ground_transform)
# sys.exit(0)
# chosen.extend(
out_sclts = match(query,
d_query,
query_2d,
sclt,
intr,
gap,
tr_ground,
scale=scale_2d)
if not len(out_sclts):
continue
# pick best from scene
chosen.append(
[out_sclts[0][i]
for i in range(len(out_sclts[0]))] + [sc_id])
# break
# os.system("rm %s/skel_%s_fill_%03d_%03d__*.json"
# % (p_out_sclts, name_query, gap[0], gap[1]))
chosen = sorted(chosen, key=lambda score_sclt: score_sclt[0])
for sid, (score, out_sclt, sc_id) in enumerate(chosen):
p_out = pjoin(
p_out_sclts, "skel_%s_fill_%03d_%03d__%02d.json" %
(name_query, gap[0], gap[1], sid))
out_sclt.save(p_out)
if sid > 5:
break
times.append(timer.get_elapsed_ms())
lg.info("mean time per gap: %s" % np.mean(times))
"""
Setup:
conda create --name iMapper python=3 numpy -y \
conda activate iMapper
Usage:
export PYTHONPATH=$(pwd); python3 example.py
"""
from imapper.logic.scenelet import Scenelet
if __name__ == '__main__':
scenelet = Scenelet.load('i3DB/Scene04/gt/skel_lobby19-3_GT.json')
skeleton = scenelet.skeleton
print('Have {} poses and {} objects'
.format(len(skeleton.get_frames()), len(scenelet.objects)))
def main(argv=None):
np.set_printoptions(suppress=True)
parser = argparse.ArgumentParser()
parser.add_argument('d', help="Folder of scene")
parser.add_argument('-resolution',
help='Target resolution for occupancy map',
default=0.1)
parser.add_argument(
'-thresh-area',
help='Ratio of occupancy map cell area that has to be occupied '
'for it to count as occupied',
default=0.1)
parser.add_argument('-postfix',
type=str,
help="Scene postfix for augmentation",
default="")
args = parser.parse_args(argv if argv is not None else sys.argv)
res_target = args.resolution
if args.postfix and len(args.postfix) and not args.postfix.startswith('_'):
args.postfix = "_%s" % args.postfix
path_parent, name_input = os.path.split(os.path.abspath(args.d))
lg.warning("name input: %s" % name_input)
path_for_tf = os.path.abspath(
os.path.join(path_parent, os.pardir, 'dataset'))
# if 'video' not in path_parent else os.path.join(path_parent, 'dataset')
if not os.path.exists(path_for_tf):
os.makedirs(path_for_tf, mode=0o0775)
lg.debug("Loading scenelet...")
path_scenelet = os.path.join(args.d, "skel_%s.json" % name_input)
scenelet = Scenelet.load(path_scenelet)
lg.debug("Scenelet: %s" % scenelet)
path_state_pickle = os.path.join(args.d, "state%s.pickle" % args.postfix)
if not os.path.exists(path_state_pickle):
lg.error("Does not exist: %s" % path_state_pickle)
return False
# assert os.path.exists(path_state_pickle), \
# "Does not exist: %s" % path_state_pickle
lg.debug("Loading volume...")
state = pickle_load(open(path_state_pickle, 'rb'))
lg.debug("Loaded volume...")
lg.debug("Creating scene from scenelet")
if not no_vis:
vis = Visualizer(win_size=(1024, 1024))
vis.add_coords()
else:
vis = None
# scene = Scene(scenelet.name_scenelet)
# colors = {0: (200., 0., 0.), 1: (0., 200., 0.), 2: (0., 0., 200.)}
# unit_x = np.array((1., 0., 0.))
occup = State(room=state.room,
tr_ground_inv=None,
res_theta=state.resolution[3],
resolution=[res_target, res_target, res_target])
occup.get_volume(labels_to_lin_ids_arg=state.get_labels_to_lin_ids())
occup_angle = np.ones(shape=(len(
occup.volume), occup.volume[0].shape[0], occup.volume[0].shape[1], 1),
dtype=np.float32) * -1.
assert np.min(occup_angle) < 0. and np.max(occup_angle) < 0., "Not empty"
grid_polys = get_grid_shapely(occup=occup, res_orig=state.resolution)
occup.volume.flags.writeable = True
volume_occp = occup.volume
angles = sorted(state.get_angles())
labels_to_lin_ids = occup.get_labels_to_lin_ids()
had_vtk_problem = no_vis
plt.figure()
rects = []
for oid, ob in scenelet.objects.items():
assert oid >= 0, "Need positive here"
label = ob.label
if label in TRANSLATIONS_CATEGORIES:
label = TRANSLATIONS_CATEGORIES[label]
if label not in labels_to_lin_ids:
continue
try:
poly = get_poly([part.obb for part in ob.parts.values()])
except ValueError as e:
print("\n===========\n\nShapely error: %s for %s\n\n" %
(e, (label, oid, ob)))
with open('error.log', 'a') as f:
f.write("[%s] %d, %s, %s\n" % (args.d, oid, label, ob))
continue
ob_angle = ob.get_angle(positive_only=True)
assert 0. <= ob_angle <= 2 * np.pi, "No: %g" % ob_angle
rect = get_rectangle(poly, ob_angle)
rect.extend([oid, CATEGORIES[label]])
rects.append(rect)
cat_id = labels_to_lin_ids[label] # cat_id in volume, not categories
for gp in grid_polys:
# skip, if not occupied enough
if gp.poly.intersection(poly).area / gp.area < args.thresh_area:
continue
# save occupancy
gp.occupancy = 1.
id_angle_lower = None
id_angle_upper = None
if ob_angle > angles[-1]:
id_angle_lower = len(angles) - 1
id_angle_upper = 0
else:
for id_angle, angle in enumerate(angles):
if ob_angle < angle:
id_angle_upper = id_angle
id_angle_lower = id_angle - 1
break
assert id_angle_lower is not None \
and id_angle_upper is not None, \
"Wrong?"
assert id_angle_upper != id_angle_lower, \
"? %s %s" % (id_angle_lower, id_angle_upper)
# cache
xy = gp.xy
# zero means empty in occupancy,
# so object ids are shifted with 1
# we need object ids to filter "untouched" objects
# in tfrecords_create
if volume_occp[cat_id, xy[0], xy[1], id_angle_lower] == 0 \
or label in CATEGORIES_DOMINANT:
volume_occp[cat_id, xy[0], xy[1], id_angle_lower] = oid + 1
if volume_occp[cat_id, xy[0], xy[1], id_angle_upper] == 0 \
or label in CATEGORIES_DOMINANT:
volume_occp[cat_id, xy[0], xy[1], id_angle_upper] = oid + 1
# angles are right now not per-category, but per-scene
# hence, an object can only overwrite, if it's usually "above"
# other objects, e.g. a table
# this is a hack for a z-test
if occup_angle[cat_id, xy[0], xy[1], 0] < 0. \
or label in CATEGORIES_DOMINANT:
occup_angle[cat_id, xy[0], xy[1], 0] = ob_angle
if not had_vtk_problem:
color = COLORS_CATEGORIES[label] if label in COLORS_CATEGORIES \
else (200., 200., 200.)
try:
for id_part, part in ob.parts.items():
vis.add_mesh(MeshOBJ.from_obb(part.obb),
name="ob_%02d_part_%02d" % (oid, id_part),
color=color)
except AttributeError:
print("VTK problem...")
had_vtk_problem = True
#plt.savefig()
plt.close()
if not had_vtk_problem:
vis.set_camera_pos(pos=(0., -1., 0.))
vis.camera().SetFocalPoint(0., 0., 0.)
vis.camera().SetViewUp(-1., 0., 0.)
vis.set_camera_type(is_ortho=True)
vis.camera().SetParallelScale(3.)
# vis.show()
name_recording = "%s_%s" % (os.path.basename(args.d), args.postfix) \
if args.postfix else os.path.basename(args.d)
lg.info("name_recording: %s" % name_recording)
path_out_occp = os.path.join(os.path.dirname(args.d), os.pardir,
'occupancy', name_recording)
if not os.path.exists(path_out_occp):
os.makedirs(path_out_occp)
# prepare www storage
www_grid = {'evidence': {}, 'occ': {}}
# normalize evidence maps
vmax = 0.
ims = {}
for cat, cat_id in labels_to_lin_ids.items():
ims[cat] = np.squeeze(
np.sum(state.volume[cat_id, :, :, :], axis=2, keepdims=True))
vmax = max(vmax, np.max(ims[cat]))
# gather joined occupancy map
im_sum = None
# for each evidence category
for cat, cat_id in labels_to_lin_ids.items():
im = ims[cat] / vmax * 255.
path_out_im = os.path.join(path_out_occp, "e_%s.jpg" % cat)
cv2.imwrite(path_out_im, im)
# lg.debug("wrote to %s" % path_out_im)
www_grid['evidence'][cat] = path_out_im
im = np.squeeze(volume_occp[cat_id, :, :, 0])
path_out_im = os.path.join(path_out_occp, "o_%s.jpg" % cat)
cv2.imwrite(path_out_im, im * 255.)
# lg.debug("wrote to %s" % path_out_im)
www_grid['occ'][cat] = path_out_im
if im_sum is None:
im_sum = im.copy()
else:
im_sum = np.maximum(im, im_sum)
#
# save dataset
#
name_input_old = name_input
if args.postfix is not None and len(args.postfix):
name_input = "%s_%s" % (name_input, args.postfix)
# state
path_state_dest = os.path.join(path_for_tf, "state_%s.pickle" % name_input)
shutil.copyfile(path_state_pickle, path_state_dest)
lg.info("Copied\n\t%s to\n\t%s" % (path_state_pickle, path_state_dest))
# occupancy
path_occup_dest = os.path.join(path_for_tf, "occup_%s.pickle" % name_input)
pickle.dump(occup, open(path_occup_dest, 'wb'), -1)
lg.info("Wrote to %s" % path_occup_dest)
# occupancy_angle
path_occup_angle_dest = os.path.join(path_for_tf,
"angle_%s.npy" % name_input)
min_angle = np.min(occup_angle)
assert min_angle < 0., "No empty cells??"
lg.debug("min angle is %s" % min_angle)
np.save(open(path_occup_angle_dest, 'wb'), occup_angle)
lg.info("Wrote to %s" % path_occup_angle_dest)
# skeleton
path_copied = shutil.copy2(path_scenelet, path_for_tf)
lg.info("Copied\n\t%s to \n\t%s" % (path_scenelet, path_copied))
# charness skeleton
name_skeleton_charness = "skel_%s-charness.json" % name_input_old
path_scenelet_charness = os.path.join(args.d, name_skeleton_charness)
assert os.path.exists(path_scenelet_charness), \
"Does not exist: %s" % path_scenelet_charness
shutil.copy2(path_scenelet_charness, path_for_tf)
assert os.path.exists(os.path.join(path_for_tf, name_skeleton_charness)), \
"Does not exist: %s" % os.path.join(path_for_tf,
name_skeleton_charness)
# rectangles
name_rectangles = "rectangles_%s.npy" % name_input_old
path_rectangles = os.path.join(path_for_tf, name_rectangles)
np.save(open(path_rectangles, 'wb'), rects)
#
# visualize
#
path_out_im = os.path.join(path_out_occp, '3d.png')
if not had_vtk_problem:
vis.save_png(path_out_im)
www_grid['3d'] = path_out_im
path_out_im = os.path.join(path_out_occp, 'o_sum.png')
max_im_sum = np.max(im_sum)
if max_im_sum > 0.:
cv2.imwrite(path_out_im, im_sum / max_im_sum * 255.)
else:
cv2.imwrite(path_out_im, im_sum * 255.)
www_grid['o_sum'] = path_out_im
path_www = os.path.join(path_out_occp, os.pardir)
with open(os.path.join(path_www, 'index.html'), 'a') as f:
f.write("<style> img {image-rendering: pixelated; } </style>\n")
f.write("<script>\n")
f.write("</script>\n")
f.write("<h3>%s</h3>" % os.path.basename(args.d))
f.write('<table>\n')
f.write("<tr>\n")
f.write("<th>3d</th>")
f.write("<th>Occupancy sum</th>")
for cat in www_grid['evidence']:
f.write("\t<th>%s</th>\n" % cat)
f.write("<th></th>\n") # titles
f.write("</tr>\n")
f.write("<tr>\n")
# 3D
f.write("\t<td rowspan=\"2\">\n")
path_im = os.path.relpath(www_grid['3d'], path_www)
f.write("\t<a href=\"%s\">\n"
"\t\t<img src=\"%s\" height=\"400\" />\n"
"\t</a>\n" % (path_im, path_im))
# Evidence sum
f.write("\t<td rowspan=\"2\">\n")
path_im = os.path.relpath(www_grid['o_sum'], path_www)
f.write("\t<a href=\"%s\">\n"
"\t\t<img src=\"%s\" height=\"400\" />\n"
"\t</a>\n" % (path_im, path_im))
# Evidence
for cat in www_grid['evidence']:
f.write("<td style=\"padding-bottom: 2px\">\n")
path_im = os.path.relpath(www_grid['evidence'][cat], path_www)
f.write("\t<a href=\"%s\">\n"
"\t\t<img src=\"%s\" height=\"200\" />\n"
"\t</a>\n" % (path_im, path_im))
f.write("</td>\n")
f.write("<td>Evidence</td>\n")
f.write("\t</td>\n")
f.write("</tr>\n")
f.write("<tr>\n")
for cat in www_grid['occ']:
f.write("<td>\n")
path_im = os.path.relpath(www_grid['occ'][cat], path_www)
f.write("\t<a href=\"%s\">\n"
"\t\t<img src=\"%s\" height=\"200\" />\n"
"</a>\n" % (path_im, path_im))
f.write("</td>\n")
f.write("<td>Occupancy map</td>\n")
f.write("</tr>")
f.write('</table>')
return True
def main(argv):
from imapper.pose.opt_consistent import main as opt_consistent
pjoin = os.path.join
parser = argparse.ArgumentParser("Fit full video")
parser.add_argument("video", type=argparse_check_exists, help="Input path")
parser.add_argument("step_size", type=int, help="Stepsize in frames.")
parser.add_argument("window_size", type=int, help="Window size in frames.")
parser.add_argument('--wp',
type=float,
help="Projection weight.",
default=1.)
parser.add_argument('--ws',
type=float,
help="Smoothness weight.",
default=.1)
parser.add_argument('--wo',
type=float,
help="Occlusion weight.",
default=0.1)
parser.add_argument('--wi',
type=float,
help="Intersection weight.",
default=1.
# used to be 10.
)
parser.add_argument('--gtol',
type=float,
help="Optimizer gradient tolerance (termination "
"criterion).",
default=1e-6)
parser.add_argument('--maxiter',
type=int,
help="Optimizer max number of iterations.",
default=0)
parser.add_argument('-w-occlusion',
action='store_true',
help="Estimate occlusion score.")
parser.add_argument('-no-isec',
action='store_true',
help='Don\'t use intersection terms')
parser.add_argument('--dest-dir',
type=str,
help="Name of subdirectory to save output to.",
default='opt1')
parser.add_argument("-s",
"--d-scenelets",
dest='s',
type=argparse_check_exists,
help="Folder containing original PiGraphs scenelets")
parser.add_argument('--batch-size',
type=int,
help="How many scenelets to optimize at once.",
default=1500)
parser.add_argument('--output-n',
type=int,
help="How many candidates to output per batch and "
"overall.",
default=200)
parser.add_argument('--filter-same-scene',
action='store_true',
help="Hold out same scene scenelets.")
args = parser.parse_args(argv)
# get video parent directory
d_query = args.video if os.path.isdir(args.video) \
else os.path.dirname(args.video)
# save call log to video directory
with open(pjoin(d_query, 'args_opt_consistent.txt'), 'a') as f_args:
f_args.write('(python3 ')
f_args.write(" ".join(sys.argv))
f_args.write(")\n")
# parse video path
name_query = os.path.split(d_query)[-1]
p_query = pjoin(d_query, "skel_%s_unannot.json" % name_query) \
if os.path.isdir(args.video) else args.video
assert p_query.endswith('.json'), "Need a skeleton file"
print("name_query: %s" % name_query)
cache_scenes = None
skipped = []
# load initial video path (local poses)
query = Scenelet.load(p_query, no_obj=True)
frame_ids = query.skeleton.get_frames()
half_window_size = args.window_size // 2
for mid_frame_id in range(frame_ids[0] + half_window_size,
frame_ids[-1] - half_window_size + 1,
args.step_size):
gap = (mid_frame_id - half_window_size,
mid_frame_id + half_window_size)
assert gap[0] >= frame_ids[0]
assert gap[1] <= frame_ids[-1]
pose_count = sum(1 for _frame_id in range(gap[0], gap[1] + 1)
if query.skeleton.has_pose(_frame_id))
if pose_count < 9:
print("Skipping span because not enough poses: %s" % pose_count)
skipped.append((gap, pose_count))
same_actor = query.skeleton.n_actors == 1 # type: bool
if not same_actor:
same_actor = query.skeleton.get_actor_id(frame_id=gap[0]) \
== query.skeleton.get_actor_id(frame_id=gap[1])
if not same_actor:
print('skipping gap {:d}...{:d}, not same actor'.format(
gap[0], gap[1]))
continue
lg.info("gap: %s" % repr(gap))
argv = [
'-silent',
'--wp',
"%g" % args.wp,
'--ws',
"%g" % args.ws,
'--wo',
"%g" % args.wo,
'--wi',
"%g" % args.wi,
'--nomocap', # added 16/4/2018
'-v',
args.video,
'--output-n',
"%d" % args.output_n
]
if args.w_occlusion:
argv.extend(['-w-occlusion'])
if args.no_isec:
argv.extend(['-no-isec'])
if args.filter_same_scene:
argv.extend(['--filter-scenes', name_query.partition('_')[0]])
# else:
# assert False, "crossvalidation assumed"
if args.maxiter:
argv.extend(['--maxiter', "%d" % args.maxiter])
argv.extend([
'independent', '-s', args.s, '--gap',
"%d" % gap[0],
"%d" % gap[1], '--dest-dir', args.dest_dir, '-tc', '-0.1',
'--batch-size',
"%d" % args.batch_size
])
lg.info("argv: %s" % argv)
# if 'once' not in locals():
try:
_cache_scenes = opt_consistent(argv, cache_scenes)
if isinstance(_cache_scenes, list) and len(_cache_scenes) \
and (cache_scenes is None
or len(_cache_scenes) != len(cache_scenes)):
cache_scenes = _cache_scenes
except FileNotFoundError as e:
lg.error("e: %s" % e)
if e.__str__().endswith('_2d_00.json\''):
from imapper.pose.main_denis import main as opt0
argv_opt0 = [
's8', '-d',
"%s/denis" % d_query, '-smooth', '0.005'
]
opt0(argv_opt0)
else:
print(e.__str__())
opt_consistent(argv)
# once = True
show_folder([args.video])
extract_gaps([args.video, args.s])
print("skipped: %s" % skipped)
def show_folder(argv):
# python3 stealth/pose/fit_full_video.py --show /home/amonszpa/workspace/stealth/data/video_recordings/scenelets/lobby15 opt1
# python3 stealth/pose/visualization/show_charness_scores.py --show /media/data/amonszpa/stealth/shared/video_recordings/library1 -o opt1
pjoin = os.path.join
parser = argparse.ArgumentParser("Fit full video")
parser.add_argument('--show', action='store_true')
parser.add_argument("video", type=argparse_check_exists, help="Input path")
parser.add_argument(
'-o',
'--opt-folder',
help="Which optimization output to process. Default: opt1",
default='opt1')
parser.add_argument("--window-size",
type=int,
help="Window size in frames.",
default=20)
args = parser.parse_args(argv)
d = os.path.join(args.video, args.opt_folder)
assert os.path.exists(d), "does not exist: %s" % d
# parse video path
if args.video.endswith(os.sep):
args.video = args.video[:-1]
name_query = os.path.split(args.video)[-1]
print("split: %s" % repr(os.path.split(args.video)))
p_query = pjoin(args.video, "skel_%s_unannot.json" % name_query) \
if os.path.isdir(args.video) else args.video
assert p_query.endswith('.json'), "Need a skeleton file"
# load initial video path (local poses)
query = Scenelet.load(p_query, no_obj=True)
frame_ids = query.skeleton.get_frames()
centroids = Skeleton.get_resampled_centroids(start=frame_ids[0],
end=frame_ids[-1],
old_frame_ids=frame_ids,
poses=query.skeleton.poses)
depths_times_charnesses = []
skeleton = Skeleton()
depths = []
skeleton.charness_poses = {} # this is in Scenelet incorrectly...
skeleton.score_fit = {} # inventing this now
skeleton.score_reproj = {} # inventing this now
for p in sorted(os.listdir(d)):
d_time = pjoin(d, p)
if not os.path.isdir(d_time):
continue
p_skel = next(
f for f in os.listdir(d_time) if os.path.isfile(pjoin(d_time, f))
and f.startswith('skel') and f.endswith('json') and '_00' in f)
sclt = Scenelet.load(pjoin(d_time, p_skel))
mn, mx = sclt.skeleton.get_frames_min_max()
frame_id = mn + (mx - mn) // 2
if query.skeleton.has_pose(frame_id):
pos_3d = query.skeleton.get_centroid_3d(frame_id)
else:
lin_id = frame_id - frame_ids[0]
pos_3d = centroids[lin_id, :]
# put centroid for each joint
skeleton.set_pose(frame_id=frame_id,
pose=np.tile(pos_3d[:, None], (1, 16)))
with open(pjoin(d_time, 'avg_charness.json')) as fch:
data = json.load(fch)
set_or_max(skeleton.charness_poses, frame_id, data['avg_charness'])
# if frame_id in skeleton.charness_poses:
# lg.warning("Maxing charness at frame %d" % frame_id)
# skeleton.charness_poses[frame_id] = max(
# skeleton.charness_poses[frame_id], data['avg_charness'])
# else:
# skeleton.charness_poses[frame_id] = data['avg_charness']
# fit scores
if 'score_fit' in sclt.aux_info:
set_or_max(skeleton.score_fit, frame_id,
sclt.aux_info['score_fit'])
else:
set_or_max(skeleton.score_fit, frame_id, 0.)
if 'score_reproj' in sclt.aux_info:
set_or_max(skeleton.score_reproj, frame_id,
sclt.aux_info['score_reproj'])
else:
set_or_max(skeleton.score_reproj, frame_id, 0.)
fig = plt.figure(figsize=(16, 12), dpi=100)
ax = fig.add_subplot(121, aspect='equal')
X = [] # skeleton x
Z = [] # skeleton z (depth)
C = [] # charness
F = [] # score_fit
R = [] # score_reproj
T = [] # times
for frame_id in skeleton.get_frames():
c = skeleton.get_joint_3d(6, frame_id=frame_id)
X.append(c[0])
Z.append(c[2])
C.append(skeleton.charness_poses[frame_id])
F.append(skeleton.score_fit[frame_id])
R.append(skeleton.score_reproj[frame_id])
T.append(frame_id)
ax.plot(X, Z, 'k--')
for frame_id in skeleton.get_frames():
if frame_id % 5:
continue
c = skeleton.get_joint_3d(6, frame_id=frame_id)
ax.annotate("%d" % frame_id, xy=(c[0], c[2]), zorder=5)
cax = ax.scatter(X, Z, c=C, cmap='jet', zorder=5)
fig.colorbar(cax)
z_lim = (min(Z), max(Z))
z_span = (z_lim[1] - z_lim[0]) // 2
x_lim = min(X), max(X)
x_span = (x_lim[1] - x_lim[0]) // 2
pad = .5
dspan = z_span - x_span
if dspan > 0:
ax.set_xlim(x_lim[0] - dspan - pad, x_lim[1] + dspan + pad)
ax.set_ylim(z_lim[0] - pad, z_lim[1] + pad)
else:
ax.set_xlim(x_lim[0] - pad, x_lim[1] + pad)
ax.set_ylim(z_lim[0] + dspan - pad, z_lim[1] - dspan + pad)
ax.set_title('Fit score weighted characteristicness\ndisplayed at '
'interpolated initial path position')
ax = fig.add_subplot(122)
ax.plot(T, C, 'x--', label='max charness')
charness_threshes = [0.4, 0.35, 0.3]
mn_thr_charness = min(charness_threshes)
mx_thr_charness = max(charness_threshes)
for ct in charness_threshes:
ax.plot([T[0], T[-1]], [ct, ct], 'r')
ax.annotate("charness %g" % ct, xy=(T[0], ct + 0.005))
charness_sorted = sorted([(fid, c)
for fid, c in skeleton.charness_poses.items()],
key=lambda e: e[1])
to_show = []
# Fitness
divisor = 5.
F_ = -np.log10(F) / divisor
print(F_)
ax.plot(T, F_, 'x--', label="-log_10(score) / %.0f" % divisor)
mx_F_ = np.percentile(F_, 90) # np.max(F_)
for i, (t, f) in enumerate(zip(T, F_)):
if f > mx_F_ or any(C[i] > ct for ct in charness_threshes):
to_show.append(i)
# ax.annotate("%.4f" % (F[i]), xy=(t, f), xytext=(t+4, f-0.02),
# arrowprops=dict(facecolor='none', shrink=0.03))
# charness
# ax.annotate("%.3f\n#%d" % (C[i], t), xy=(t, C[i]),
# xytext=(t-10, C[i]-0.02),
# arrowprops=dict(facecolor='none', shrink=0.03))
windows = [] # [(t_start, t_max, t_end), ...]
crossings = {}
# Reproj
R_ = -np.log10(R) / divisor
# ax.plot(T, R_, 'x--', label="-log_10(score reproj) / %.0f" % divisor)
mx_R_ = np.max(R_)
is_above = [False for _ in charness_threshes]
mx_above = []
for i, (t, r) in enumerate(zip(T, R_)):
# if i in to_show:
# ax.annotate("%.4f" % (R[i]), xy=(t, r), xytext=(t-10, r+0.02),
# arrowprops=dict(facecolor='none', shrink=0.03))
# ax.annotate("%d" % t, xy=(t, r - 0.01))
if (i + 1 < len(C)) and (C[i] > C[i + 1]) and (C[i] > mn_thr_charness):
mx_above.append((C[i], t))
for thr_i, thr in enumerate(charness_threshes):
if (C[i] > thr) != is_above[thr_i] \
or (C[i] > mx_thr_charness and not is_above[thr_i]):
step = 15 * (len(charness_threshes) - thr_i) \
if is_above[thr_i] \
else -15 * thr_i
if is_above[thr_i]:
if 'down' not in crossings:
crossings['down'] = (C[i], t)
# else:
# assert crossings['down'][0] > C[i], (crossings['down'][0], C[i])
else:
if 'up' not in crossings:
crossings['up'] = (C[i - 1], t)
elif crossings['up'][0] < C[i - 1]:
crossings['up'] = (C[i - 1], t)
# ax.annotate("%.3f\n#%d" % (C[i], t), xy=(t, C[i]),
# xytext=(t + step, C[i]-0.1),
# arrowprops=dict(facecolor='none', shrink=0.03))
if C[i] < mn_thr_charness and is_above[thr_i]:
try:
c, t = max((e for e in mx_above), key=lambda e: e[0])
ax.annotate("%.3f\n#%d" % (c, t),
xy=(t, c),
xytext=(t + step, c + 0.1),
arrowprops=dict(facecolor='none',
shrink=0.03))
mx_above = []
windows.append(
(crossings['up'][1], t, crossings['down'][1]))
except (KeyError, ValueError):
lg.warning("Can't find gap: %s, %s" %
(crossings, mx_above))
crossings = {}
is_above[thr_i] = C[i] > thr
break
for crossing in windows:
for i, t in enumerate(crossing):
c = skeleton.charness_poses[t]
step = -15 + i * 10
ax.annotate("%.3f\n#%d" % (c, t),
xy=(t, c),
xytext=(t + step, c - 0.1),
arrowprops=dict(facecolor='none', shrink=0.03))
# extract_gaps([args.video])
# labels
ax.set_title("Scores and charness w.r.t time: max charness: #%d %g" %
(charness_sorted[-1][0], charness_sorted[-1][1]))
ax.set_xlabel('integer time')
ax.legend(loc='lower right')
ax.grid(True)
ax.yaxis.grid(which='both')
ax.xaxis.set_ticks(np.arange(T[0] - 1, T[-1] + 1, 5))
ax.set_yticks([])
ax.set_ylim(0., 1.)
ax.set_ylabel('higher is better')
plt.suptitle("%s" % name_query)
with open(os.path.join(d, 'charness_rank.csv'), 'w') as fout:
fout.write("frame_id,charness\n")
for fid_charness in reversed(charness_sorted):
fout.write("{:d},{:g}\n".format(*fid_charness))
print(fid_charness)
# plt.show()
p_out = os.path.join(d, 'charnesses.svg')
plt.savefig(p_out)
lg.debug("saved to %s" % p_out)
def prepare(camera_name, winsorize_limit, shape_orig, path_scene, skel_ours_2d,
skel_ours, resample, path_skel):
"""
Args:
camera_name (str):
Name of camera for intrinsics calculation.
winsorize_limit (float):
Outlier detection threshold.
shape_orig (Tuple[int, int]):
Original video resolution.
path_scene (str): Root path to scene.
skel_ours_2d (np.ndarray): (N, 2, 16)
2D skeletons from LFD in our format.
skel_ours (np.ndarray): (N, 3, 16)
Local space 3D skeletons in iMapper coordinate frame
(y-down, z-front).
resample (bool):
If needs densification using Blender's IK engine.
Returns:
skel_ours (Skeleton):
skel_ours_2d (Skeleton):
intrinsics (np.ndarray):
"""
assert camera_name is not None and isinstance(camera_name, str), \
"Need a camera name"
if shape_orig is None:
shape_orig = (np.float32(1080.), np.float32(1920.))
np.set_printoptions(linewidth=200, suppress=True)
if False:
plt.figure()
for i, frame_id in enumerate(skel_ours.get_frames()):
plot_2d(skel_ours_2d.get_pose(frame_id), images[frame_id])
plt.show()
path_intrinsics = os.path.join(path_scene, "intrinsics.json")
if os.path.exists(path_intrinsics):
lg.warning("Loading existing intrinsics matrix!")
K = np.array(json.load(open(path_intrinsics, 'r')), dtype=np.float32)
scale = (shape_orig[1] /
int(round(shape_orig[1] * float(INPUT_SIZE) / shape_orig[0])),
shape_orig[0] / float(INPUT_SIZE))
K[0, 0] /= scale[0]
K[0, 2] /= scale[0]
K[1, 1] /= scale[1]
K[1, 2] /= scale[1]
else:
K = intrinsics_matrix(INPUT_SIZE, shape_orig, camera_name)
focal_correction = Conf.get().optimize_path.focal_correction
if abs(focal_correction - 1.) > 1.e-3:
lg.warning("Warning, scaling intrinsics matrix by %f" %
focal_correction)
K[0, 0] *= focal_correction
K[1, 1] *= focal_correction
#print("K:\n%s,\nintr:\n%s" % (K, intr))
# sys.exit(0)
#
# Prune poses
#
skel_ours_2d, frame_ids_removed = filter_outliers(
skel_ours_2d, winsorize_limit=winsorize_limit, show=False)
frames_to_remove_3d = filter_wrong_poses(skel_ours_2d, skel_ours)
frames_to_ignore_list = set()
# if frames_ignore is not None:
# for start_end in frames_ignore:
# if isinstance(start_end, tuple):
# l_ = list(range(
# start_end[0],
# min(start_end[1], skel_ours_2d.get_frames()[-1])))
# frames_to_remove_3d.extend(l_)
# frames_to_ignore_list.update(l_)
# else:
# assert isinstance(start_end, int), \
# "Not int? %s" % repr(start_end)
# frames_to_remove_3d.append(start_end)
# frames_to_ignore_list.add(start_end)
for frame_id in skel_ours.get_frames():
if frame_id in frames_to_remove_3d:
skel_ours.remove_pose(frame_id)
# resample skeleton to fill in missing frames
skel_ours_old = skel_ours
frame_ids_filled_in = set(skel_ours_2d.get_frames()).difference(
set(skel_ours_old.get_frames()))
if resample:
lg.warning("Resampling BEFORE optimization")
# frames_to_resample = sorted(set(skel_ours_2d.get_frames()).difference(
# frames_to_ignore_list))
# skel_ours = Skeleton.resample(skel_ours_old,
# frame_ids=frames_to_resample)
# Aron on 6/4/2018
sclt_ours = Scenelet(skeleton=skel_ours)
stem = os.path.splitext(path_skel)[0]
path_filtered = "%s_filtered.json" % stem
path_ipoled = "%s_ikipol.json" % os.path.splitext(path_filtered)[0]
if not os.path.exists(path_ipoled):
sclt_ours.save(path_filtered)
script_filepath = \
os.path.normpath(os.path.join(
os.path.dirname(os.path.abspath(__file__)),
os.pardir, 'blender', 'ipol_ik.py'))
assert os.path.exists(script_filepath), "No: %s" % script_filepath
blender_path = os.environ.get('BLENDER')
if not os.path.isfile(blender_path):
raise RuntimeError(
"Need \"BLENDER\" environment variable to be set "
"to the blender executable")
cmd_params = [
blender_path, '-noaudio', '-b', '-P', script_filepath, '--',
path_filtered
]
print("calling %s" % " ".join(cmd_params))
ret = check_call(cmd_params)
print("ret: %s" % ret)
else:
lg.warning("\n\n\tNOT recomputing IK interpolation, "
"file found at %s!\n" % path_ipoled)
skel_ours = Scenelet.load(path_ipoled, no_obj=True).skeleton
# remove extra frames at ends and beginnings of actors
spans = skel_ours_old.get_actor_empty_frames()
old_frames = skel_ours_old.get_frames()
frames_to_remove = []
for frame_id in skel_ours.get_frames():
if frame_id not in old_frames:
in_spans = next(
(True for span in spans if span[0] < frame_id < span[1]),
None)
if in_spans:
frames_to_remove.append(frame_id)
# lg.debug("diff: %s (a%s, f%s)"
# % (
# frame_id,
# skel_ours_old.get_actor_id(frame_id),
# skel_ours_old.mod_frame_id(frame_id)
# ))
for frame_id in frames_to_remove:
skel_ours.remove_pose(frame_id)
for frame_id in skel_ours_2d.get_frames():
if not skel_ours.has_pose(frame_id):
skel_ours_2d.remove_pose(frame_id)
for frame_id in skel_ours.get_frames():
if not skel_ours_2d.has_pose(frame_id):
skel_ours.remove_pose(frame_id)
frames_set_ours = set(skel_ours.get_frames())
frames_set_2d = set(skel_ours_2d.get_frames())
if frames_set_ours != frames_set_2d:
print("Frame mismatch: %s" % frames_set_ours.difference(frames_set_2d))
lg.warning("Removing pelvis and neck from 2D input")
for frame_id in skel_ours_2d.get_frames():
skel_ours_2d.set_visible(frame_id, Joint.PELV, 0)
skel_ours_2d.set_visible(frame_id, Joint.NECK, 0)
return skel_ours, skel_ours_2d, K, frame_ids_filled_in
def evaluate(named_solution,
sclt_gt,
sclt_gt_2d,
frame_ids,
path_scene,
stats,
actions=None,
scale=100.):
"""
:param named_solution:
:param sclt_gt:
:param sclt_gt_2d:
:param frame_ids:
:param path_scene:
:param stats:
:param actions:
:param scale: scale from meter to cm
:return:
"""
p_intrinsics = os.path.join(path_scene, 'intrinsics.json')
intrinsics = np.array(json.load(open(p_intrinsics, 'r')), dtype=np.float32)
print('Loading {}'.format(named_solution.path))
sclt_sol = Scenelet.load(named_solution.path)
sclt_sol.skeleton._visibility.clear()
sclt_sol.skeleton._confidence.clear()
sclt_sol.skeleton._forwards.clear()
sclt_sol.skeleton = Skeleton.resample(sclt_sol.skeleton)
err_3d = Comparison(title='World',
path_scene=path_scene,
name_method=named_solution.name_method,
stats=stats)
err_3d_local = Comparison(title='Local',
path_scene=path_scene,
name_method=named_solution.name_method,
stats=stats)
err_2d = Comparison(title='2D',
path_scene=path_scene,
name_method=named_solution.name_method,
stats=stats)
occlusion = sclt_gt.aux_info['occluded']
missing = {'method': [], 'gt': []}
for frame_id in frame_ids:
try:
entry = sclt_sol.skeleton.get_pose(frame_id=frame_id)
except KeyError:
missing['method'].append(frame_id)
continue
if actions is not None and frame_id in actions \
and actions[frame_id] == 'walking':
print('Skipping non-interactive frame {} {}'.format(
frame_id, actions[frame_id]))
continue
# 3D
gt = sclt_gt.skeleton.get_pose(frame_id=frame_id)
occluded = occlusion['{:d}'.format(frame_id)]
err_3d.add(entry=entry,
gt=gt,
frame_id=frame_id,
scale=scale,
occluded=occluded)
# Local 3D
local_entry = entry - entry[:, Joint.PELV:Joint.PELV + 1]
local_gt = gt - gt[:, Joint.PELV:Joint.PELV + 1]
err_3d_local.add(entry=local_entry,
gt=local_gt,
frame_id=frame_id,
scale=scale,
occluded=occluded)
#
# GT 2D
#
gt_2d = sclt_gt_2d.skeleton.get_pose(frame_id=frame_id)
entry_2d = entry[:2, :] / entry[2, :]
entry_2d[0, :] *= intrinsics[0, 0]
entry_2d[1, :] *= intrinsics[1, 1]
entry_2d[0, :] += intrinsics[0, 2]
entry_2d[1, :] += intrinsics[1, 2]
err_2d.add(entry=entry_2d,
gt=gt_2d[:2, :],
frame_id=frame_id,
occluded=occluded)
# stats.paint(path_dest=os.path.join(path_scene, 'debug_eval'))
mn, mx = np.min(sclt_gt.skeleton.poses, axis=(0, 2)), \
np.max(sclt_gt.skeleton.poses, axis=(0, 2))
err_3d.dimensions = (mx - mn) * scale
assert len(missing['method']) < len(frame_ids) / 2, (missing, frame_ids)
return OrderedDict({
err_3d.title: err_3d,
err_3d_local.title: err_3d_local,
err_2d.title: err_2d,
'_missing': missing
})
def work_scene(p_root, plot_fun_name, name_scene, fname):
pjoin = os.path.join
name_scene = os.path.split(p_root)[-1]
p_actions = os.path.join(p_root, 'gt', 'actions.txt')
actions = None
# if os.path.isfile(p_actions):
# with open(p_actions, 'r') as f:
# actions = [line.strip().split('\t') for line in f.readlines()]
# actions = {int(line[0]): line[1] for line in actions}
p_quant2 = pjoin(p_root, _QUANT_FOLDER)
if not os.path.isdir(p_quant2):
os.makedirs(p_quant2)
p_methods = []
#
# Tome3D
#
pfixes = [('Tome3D-nosmooth', '0')] # ('Tome3D-smooth', '10')
for postfix, smooth in pfixes:
p_tome3d = pjoin(p_quant2,
'skel_{}_{}.json'.format(name_scene, postfix))
p_tome_skels = pjoin(p_root, 'denis')
if os.path.isfile(p_tome3d):
p_methods.append(NamedSolution('\\tomethreed', p_tome3d))
else:
print('Can\'t find Tome3D at {}'.format(p_tome3d))
#
# LCR-Net 3D
#
postfix = 'LCRNet3D-nosmooth'
p_lcrnet3d = pjoin(p_quant2, 'skel_{}_{}.json'.format(name_scene, postfix))
if os.path.isfile(p_lcrnet3d):
p_methods.append(
NamedSolution('\\lcrnetthreed$_{no smooth}$', p_lcrnet3d))
else:
print('Can\'t find Tome3D at {}'.format(p_tome3d))
#
# GT
#
p_gt = pjoin(p_root, 'gt', 'skel_{}_GT.json'.format(name_scene))
assert os.path.isfile(p_gt), 'Need gt file: {}'.format(p_gt)
sclt_gt = Scenelet.load(p_gt)
p_gt_2d = pjoin(p_root, 'gt', 'skel_GT_2d.json')
sclt_gt_2d = Scenelet.load(p_gt_2d)
#
# Evaluate
#
p_methods.append(NamedSolution('\\name', pjoin(p_root, 'output', fname)))
# Append your solution here
p_methods.append(
NamedSolution('NewMethod', pjoin(p_root, 'NewMethod', fname)))
stats = StatsOverTime(plot_fun_name=plot_fun_name, name_scene=name_scene)
errors = OrderedDict()
for named_solution in p_methods:
frame_ids = [
fid for fid in sclt_gt.skeleton.get_frames() if not fid % 2
]
# print(frame_ids)
errors_ = evaluate(named_solution=named_solution,
sclt_gt=sclt_gt,
sclt_gt_2d=sclt_gt_2d,
frame_ids=frame_ids,
path_scene=p_root,
stats=stats,
actions=actions)
errors[named_solution.name_method] = errors_
series = stats.paint(path_dest=os.path.join(p_root, 'debug_eval'))
return errors, series